Index: Python/ast.c =================================================================== --- Python/ast.c (revision 61862) +++ Python/ast.c (working copy) @@ -20,6 +20,7 @@ char *c_encoding; /* source encoding */ PyArena *c_arena; /* arena for allocating memeory */ const char *c_filename; /* filename */ + int c_unicode_strings; /* bool: should all strings be unicode */ }; static asdl_seq *seq_for_testlist(struct compiling *, const node *); @@ -200,6 +201,8 @@ } c.c_arena = arena; c.c_filename = filename; + c.c_unicode_strings = flags && flags->cf_flags & CO_FUTURE_UNICODE_STRINGS; +printf("uni strs: %d\n", c.c_unicode_strings); k = 0; switch (TYPE(n)) { Index: Python/pythonrun.c =================================================================== --- Python/pythonrun.c (revision 61862) +++ Python/pythonrun.c (working copy) @@ -750,7 +750,9 @@ ((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \ PyPARSE_DONT_IMPLY_DEDENT : 0) \ | ((flags)->cf_flags & CO_FUTURE_PRINT_FUNCTION ? \ - PyPARSE_PRINT_IS_FUNCTION : 0)) : 0) + PyPARSE_PRINT_IS_FUNCTION : 0) \ + | ((flags)->cf_flags & CO_FUTURE_UNICODE_STRINGS ? \ + PyPARSE_UNICODE_STRINGS : 0)) : 0) #endif int @@ -1379,6 +1381,38 @@ return st; } +static void update_flags_from_node(const node *n, PyCompilerFlags *flags) +{ + const node *import_stmt = n, *child; + while (n && n->n_nchildren != 0) { + import_stmt = n; + n = n->n_child; + } + if (import_stmt == NULL || import_stmt->n_nchildren != 4) + return; + child = &import_stmt->n_child[0]; + if (child->n_str == NULL || strcmp(child->n_str, "from") != 0) + return; + child = &import_stmt->n_child[2]; + if (child->n_str == NULL || strcmp(child->n_str, "import") != 0) + return; + child = &import_stmt->n_child[1]; + if (child->n_nchildren != 1 || + child->n_child[0].n_str == NULL || + strcmp(child->n_child[0].n_str, "__future__") != 0) + return; + /* TODO(nnorwitz): Handle string in any position. */ + child = &import_stmt->n_child[3]; + if (child->n_nchildren != 1) + return; + child = &child->n_child[0]; + if (child->n_nchildren != 1 || + child->n_child[0].n_str == NULL || + strcmp(child->n_child[0].n_str, "unicode_strings") != 0) + return; + flags->cf_flags |= CO_FUTURE_UNICODE_STRINGS; +} + /* Preferred access to parser is through AST. */ mod_ty PyParser_ASTFromString(const char *s, const char *filename, int start, @@ -1390,6 +1424,7 @@ &_PyParser_Grammar, start, &err, PARSER_FLAGS(flags)); if (n) { + update_flags_from_node(n, flags); mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); return mod; @@ -1407,9 +1442,12 @@ { mod_ty mod; perrdetail err; - node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, - start, ps1, ps2, &err, PARSER_FLAGS(flags)); + int iflags = PARSER_FLAGS(flags); + node *n = PyParser_ParseFileFlagsEx(fp, filename, &_PyParser_Grammar, + start, ps1, ps2, &err, &iflags); if (n) { + if (flags && iflags & CO_FUTURE_UNICODE_STRINGS) + flags->cf_flags |= CO_FUTURE_UNICODE_STRINGS; mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); return mod; Index: Python/future.c =================================================================== --- Python/future.c (revision 61862) +++ Python/future.c (working copy) @@ -35,6 +35,8 @@ ff->ff_features |= CO_FUTURE_WITH_STATEMENT; } else if (strcmp(feature, FUTURE_PRINT_FUNCTION) == 0) { ff->ff_features |= CO_FUTURE_PRINT_FUNCTION; + } else if (strcmp(feature, FUTURE_UNICODE_STRINGS) == 0) { + ff->ff_features |= CO_FUTURE_UNICODE_STRINGS; } else if (strcmp(feature, "braces") == 0) { PyErr_SetString(PyExc_SyntaxError, "not a chance"); Index: Include/code.h =================================================================== --- Include/code.h (revision 61862) +++ Include/code.h (working copy) @@ -49,6 +49,7 @@ #define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */ #define CO_FUTURE_WITH_STATEMENT 0x8000 #define CO_FUTURE_PRINT_FUNCTION 0x10000 +#define CO_FUTURE_UNICODE_STRINGS 0x20000 /* This should be defined if a future statement modifies the syntax. For example, when a keyword is added. Index: Include/parsetok.h =================================================================== --- Include/parsetok.h (revision 61862) +++ Include/parsetok.h (working copy) @@ -28,6 +28,7 @@ #endif #define PyPARSE_PRINT_IS_FUNCTION 0x0004 +#define PyPARSE_UNICODE_STRINGS 0x0008 @@ -41,6 +42,9 @@ PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, grammar *, int, char *, char *, perrdetail *, int); +PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *, grammar *, + int, char *, char *, + perrdetail *, int *); PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *, const char *, Index: Include/pythonrun.h =================================================================== --- Include/pythonrun.h (revision 61862) +++ Include/pythonrun.h (working copy) @@ -8,7 +8,8 @@ #endif #define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \ - CO_FUTURE_WITH_STATEMENT|CO_FUTURE_PRINT_FUNCTION) + CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \ + CO_FUTURE_UNICODE_STRINGS) #define PyCF_MASK_OBSOLETE (CO_NESTED) #define PyCF_SOURCE_IS_UTF8 0x0100 #define PyCF_DONT_IMPLY_DEDENT 0x0200 Index: Include/compile.h =================================================================== --- Include/compile.h (revision 61862) +++ Include/compile.h (working copy) @@ -25,6 +25,7 @@ #define FUTURE_ABSOLUTE_IMPORT "absolute_import" #define FUTURE_WITH_STATEMENT "with_statement" #define FUTURE_PRINT_FUNCTION "print_function" +#define FUTURE_UNICODE_STRINGS "unicode_strings" struct _mod; /* Declare the existence of this type */ Index: Parser/parsetok.c =================================================================== --- Parser/parsetok.c (revision 61862) +++ Parser/parsetok.c (working copy) @@ -14,7 +14,7 @@ /* Forward */ -static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int); +static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *); static void initerr(perrdetail *err_ret, const char* filename); /* Parse input coming from a string. Return error code, print some errors. */ @@ -53,7 +53,7 @@ tok->alterror++; } - return parsetok(tok, g, start, err_ret, flags); + return parsetok(tok, g, start, err_ret, &flags); } /* Parse input coming from a file. Return error code, print some errors. */ @@ -86,6 +86,29 @@ } + return parsetok(tok, g, start, err_ret, &flags); +} + +node * +PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, grammar *g, int start, + char *ps1, char *ps2, perrdetail *err_ret, int *flags) +{ + struct tok_state *tok; + + initerr(err_ret, filename); + + if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) { + err_ret->error = E_NOMEM; + return NULL; + } + tok->filename = filename; + if (Py_TabcheckFlag || Py_VerboseFlag) { + tok->altwarning = (filename != NULL); + if (Py_TabcheckFlag >= 2) + tok->alterror++; + } + + return parsetok(tok, g, start, err_ret, flags); } @@ -110,7 +133,7 @@ static node * parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, - int flags) + int *flags) { parser_state *ps; node *n; @@ -123,8 +146,12 @@ return NULL; } #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD - if (flags & PyPARSE_PRINT_IS_FUNCTION) + if (*flags & PyPARSE_PRINT_IS_FUNCTION) ps->p_flags |= CO_FUTURE_PRINT_FUNCTION; + if (*flags & PyPARSE_UNICODE_STRINGS) { + ps->p_flags |= CO_FUTURE_UNICODE_STRINGS; + printf("parsetok: uni\n"); +} #endif for (;;) { @@ -147,7 +174,7 @@ except if a certain flag is given -- codeop.py uses this. */ if (tok->indent && - !(flags & PyPARSE_DONT_IMPLY_DEDENT)) + !(*flags & PyPARSE_DONT_IMPLY_DEDENT)) { tok->pendin = -tok->indent; tok->indent = 0; @@ -191,6 +218,7 @@ else n = NULL; + *flags = ps->p_flags; PyParser_Delete(ps); if (n == NULL) { Index: Parser/parser.c =================================================================== --- Parser/parser.c (revision 61862) +++ Parser/parser.c (working copy) @@ -210,6 +210,11 @@ strcmp(STR(CHILD(cch, 0)), "print_function") == 0) { ps->p_flags |= CO_FUTURE_PRINT_FUNCTION; break; + } else if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME && + strcmp(STR(CHILD(cch, 0)), "unicode_strings") == 0) { + ps->p_flags |= CO_FUTURE_UNICODE_STRINGS; + printf("parser: uni\n"); + break; } } } Index: Lib/__future__.py =================================================================== --- Lib/__future__.py (revision 61862) +++ Lib/__future__.py (working copy) @@ -54,6 +54,7 @@ "absolute_import", "with_statement", "print_function", + "unicode_strings", ] __all__ = ["all_feature_names"] + all_feature_names @@ -68,6 +69,7 @@ CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function +CO_FUTURE_UNICODE_STRINGS = 0x20000 # all strings are unicode class _Feature: def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): @@ -120,3 +122,7 @@ print_function = _Feature((2, 6, 0, "alpha", 2), (3, 0, 0, "alpha", 0), CO_FUTURE_PRINT_FUNCTION) + +unicode_strings = _Feature((2, 6, 0, "alpha", 2), + (3, 0, 0, "alpha", 0), + CO_FUTURE_UNICODE_STRINGS)