Index: Python/pythonrun.c =================================================================== --- Python/pythonrun.c (Revision 86724) +++ Python/pythonrun.c (Arbeitskopie) @@ -1797,8 +1797,8 @@ } PyObject * -Py_CompileStringFlags(const char *str, const char *filename, int start, - PyCompilerFlags *flags) +Py_CompileStringExFlags(const char *str, const char *filename, int start, + PyCompilerFlags *flags, int optimize) { PyCodeObject *co; mod_ty mod; @@ -1816,7 +1816,7 @@ PyArena_Free(arena); return result; } - co = PyAST_Compile(mod, filename, flags, arena); + co = PyAST_CompileEx(mod, filename, flags, optimize, arena); PyArena_Free(arena); return (PyObject *)co; } @@ -2446,9 +2446,17 @@ PyAPI_FUNC(PyObject *) Py_CompileString(const char *str, const char *p, int s) { - return Py_CompileStringFlags(str, p, s, NULL); + return Py_CompileStringExFlags(str, p, s, NULL, -1); } +#undef Py_CompileStringFlags +PyAPI_FUNC(PyObject *) +Py_CompileStringFlags(const char *str, const char *p, int s, + PyCompilerFlags *flags) +{ + return Py_CompileStringExFlags(str, p, s, flags, -1); +} + #undef PyRun_InteractiveOne PyAPI_FUNC(int) PyRun_InteractiveOne(FILE *f, const char *p) Index: Python/compile.c =================================================================== --- Python/compile.c (Revision 86724) +++ Python/compile.c (Arbeitskopie) @@ -141,6 +141,7 @@ PyFutureFeatures *c_future; /* pointer to module's __future__ */ PyCompilerFlags *c_flags; + int c_optimize; /* optimization level */ int c_interactive; /* true if in interactive mode */ int c_nestlevel; @@ -177,7 +178,7 @@ static int compiler_in_loop(struct compiler *); static int inplace_binop(struct compiler *, operator_ty); -static int expr_constant(expr_ty e); +static int expr_constant(struct compiler *, expr_ty); static int compiler_with(struct compiler *, stmt_ty); static int compiler_call_helper(struct compiler *c, int n, @@ -256,8 +257,8 @@ } PyCodeObject * -PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, - PyArena *arena) +PyAST_CompileEx(mod_ty mod, const char *filename, PyCompilerFlags *flags, + int optimize, PyArena *arena) { struct compiler c; PyCodeObject *co = NULL; @@ -285,6 +286,7 @@ c.c_future->ff_features = merged; flags->cf_flags = merged; c.c_flags = flags; + c.c_optimize = (optimize == -1) ? Py_OptimizeFlag : optimize; c.c_nestlevel = 0; c.c_st = PySymtable_Build(mod, filename, c.c_future); @@ -1151,7 +1153,7 @@ if (!asdl_seq_LEN(stmts)) return 1; st = (stmt_ty)asdl_seq_GET(stmts, 0); - if (compiler_isdocstring(st) && Py_OptimizeFlag < 2) { + if (compiler_isdocstring(st) && c->c_optimize < 2) { /* don't generate docstrings if -OO */ i = 1; VISIT(c, expr, st->v.Expr.value); @@ -1465,7 +1467,7 @@ st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, 0); docstring = compiler_isdocstring(st); - if (docstring && Py_OptimizeFlag < 2) + if (docstring && c->c_optimize < 2) first_const = st->v.Expr.value->v.Str.s; if (compiler_add_o(c, c->u->u_consts, first_const) < 0) { compiler_exit_scope(c); @@ -1699,7 +1701,7 @@ if (end == NULL) return 0; - constant = expr_constant(s->v.If.test); + constant = expr_constant(c, s->v.If.test); /* constant = 0: "if 0" * constant = 1: "if 1", "if 2", ... * constant = -1: rest */ @@ -1761,7 +1763,7 @@ compiler_while(struct compiler *c, stmt_ty s) { basicblock *loop, *orelse, *end, *anchor = NULL; - int constant = expr_constant(s->v.While.test); + int constant = expr_constant(c, s->v.While.test); if (constant == 0) { if (s->v.While.orelse) @@ -2213,7 +2215,7 @@ static PyObject *assertion_error = NULL; basicblock *end; - if (Py_OptimizeFlag) + if (c->c_optimize) return 1; if (assertion_error == NULL) { assertion_error = PyUnicode_InternFromString("AssertionError"); @@ -3013,7 +3015,7 @@ */ static int -expr_constant(expr_ty e) +expr_constant(struct compiler *c, expr_ty e) { char *id; switch (e->kind) { @@ -3031,7 +3033,7 @@ if (strcmp(id, "False") == 0) return 0; if (strcmp(id, "None") == 0) return 0; if (strcmp(id, "__debug__") == 0) - return ! Py_OptimizeFlag; + return ! c->c_optimize; /* fall through */ default: return -1; @@ -4082,3 +4084,13 @@ assemble_free(&a); return co; } + +#undef PyAST_Compile +PyAPI_FUNC(PyCodeObject *) +PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, + PyArena *arena) +{ + return PyAST_CompileEx(mod, filename, flags, -1, arena); +} + + Index: Python/bltinmodule.c =================================================================== --- Python/bltinmodule.c (Revision 86724) +++ Python/bltinmodule.c (Arbeitskopie) @@ -530,19 +530,20 @@ int mode = -1; int dont_inherit = 0; int supplied_flags = 0; + int optimize = -1; int is_ast; PyCompilerFlags cf; PyObject *cmd; static char *kwlist[] = {"source", "filename", "mode", "flags", - "dont_inherit", NULL}; + "dont_inherit", "optimize", NULL}; int start[] = {Py_file_input, Py_eval_input, Py_single_input}; PyObject *result; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&s|ii:compile", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&s|iii:compile", kwlist, &cmd, PyUnicode_FSConverter, &filename_obj, &startstr, &supplied_flags, - &dont_inherit)) + &dont_inherit, &optimize)) return NULL; filename = PyBytes_AS_STRING(filename_obj); @@ -557,6 +558,12 @@ } /* XXX Warn if (supplied_flags & PyCF_MASK_OBSOLETE) != 0? */ + if (optimize < -1 || optimize > 2) { + PyErr_SetString(PyExc_ValueError, + "compile(): invalid optimize value"); + goto error; + } + if (!dont_inherit) { PyEval_MergeCompilerFlags(&cf); } @@ -591,8 +598,8 @@ PyArena_Free(arena); goto error; } - result = (PyObject*)PyAST_Compile(mod, filename, - &cf, arena); + result = (PyObject*)PyAST_CompileEx(mod, filename, + &cf, optimize, arena); PyArena_Free(arena); } goto finally; @@ -602,7 +609,7 @@ if (str == NULL) goto error; - result = Py_CompileStringFlags(str, filename, start[mode], &cf); + result = Py_CompileStringExFlags(str, filename, start[mode], &cf, optimize); goto finally; error: Index: Include/pythonrun.h =================================================================== --- Include/pythonrun.h (Revision 86724) +++ Include/pythonrun.h (Arbeitskopie) @@ -64,9 +64,10 @@ PyObject *, PyObject *, int, PyCompilerFlags *); -#define Py_CompileString(str, p, s) Py_CompileStringFlags(str, p, s, NULL) -PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *, const char *, int, - PyCompilerFlags *); +#define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1) +#define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1) +PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(const char *, const char *, int, + PyCompilerFlags *, int); PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int); PyAPI_FUNC(void) PyErr_Print(void); Index: Include/compile.h =================================================================== --- Include/compile.h (Revision 86724) +++ Include/compile.h (Arbeitskopie) @@ -29,8 +29,9 @@ #define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL" struct _mod; /* Declare the existence of this type */ -PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *, - PyCompilerFlags *, PyArena *); +#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar) +PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(struct _mod *, const char *, + PyCompilerFlags *, int, PyArena *); PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *); Index: Misc/NEWS =================================================================== --- Misc/NEWS (Revision 86724) +++ Misc/NEWS (Arbeitskopie) @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- Provide an *optimize* parameter in the built-in compile() function. + - Issue #10474: range().count() should return integers. - Issue #10255: Fix reference leak in Py_InitializeEx(). Patch by Neil Index: Doc/c-api/veryhigh.rst =================================================================== --- Doc/c-api/veryhigh.rst (Revision 86724) +++ Doc/c-api/veryhigh.rst (Arbeitskopie) @@ -230,6 +230,12 @@ .. c:function:: PyObject* Py_CompileStringFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags) + This is a simplified interface to :c:func:`Py_CompileStringExFlags` below, with + *optimize* set to ``-1``. + + +.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) + Parse and compile the Python source code in *str*, returning the resulting code object. The start token is given by *start*; this can be used to constrain the code which can be compiled and should be :const:`Py_eval_input`, @@ -238,7 +244,15 @@ :exc:`SyntaxError` exception messages. This returns *NULL* if the code cannot be parsed or compiled. + The integer *optimize* specifies the optimization level of the compiler; a + value of ``-1`` selects the optimization level of the interpreter as given by + :option:`-O` options. Explicit levels are ``0`` (no optimization; + ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) + or ``2`` (docstrings are removed too). + .. versionadded:: 3.2 + + .. c:function:: PyObject* PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals) This is a simplified interface to :c:func:`PyEval_EvalCodeEx`, with just Index: Doc/library/py_compile.rst =================================================================== --- Doc/library/py_compile.rst (Revision 86724) +++ Doc/library/py_compile.rst (Arbeitskopie) @@ -22,7 +22,7 @@ Exception raised when an error occurs while attempting to compile the file. -.. function:: compile(file, cfile=None, dfile=None, doraise=False) +.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1) Compile a source file to byte-code and write out the byte-code cache file. The source code is loaded from the file name *file*. The byte-code is written to @@ -37,7 +37,14 @@ returns the path to byte-compiled file, i.e. whatever *cfile* value was used. + *optimze* controls the optimization level and is passed to the built-in + :func:`compile` function. The default of ``-1`` selects the optimization + level of the current interpreter. + .. versionchanged:: 3.2 + Added the *optimze* parameter. + + .. function:: main(args=None) Compile several source files. The files named in *args* (or on the command Index: Doc/library/functions.rst =================================================================== --- Doc/library/functions.rst (Revision 86724) +++ Doc/library/functions.rst (Arbeitskopie) @@ -143,7 +143,7 @@ type hierarchy in :ref:`types`. -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False) +.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) Compile the *source* into a code or AST object. Code objects can be executed by :func:`exec` or :func:`eval`. *source* can either be a string or an AST @@ -175,6 +175,12 @@ can be found as the :attr:`compiler_flag` attribute on the :class:`_Feature` instance in the :mod:`__future__` module. + The argument *optimize* specifies the optimization level of the compiler; the + default value of ``-1`` selects the optimization level of the interpreter as + given by :option:`-O` options. Explicit levels are ``0`` (no optimization; + ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) + or ``2`` (docstrings are removed too). + This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`TypeError` if the source contains null bytes. @@ -187,7 +193,7 @@ .. versionchanged:: 3.2 Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode - does not have to end in a newline anymore. + does not have to end in a newline anymore. Added the *optimize* parameter. .. function:: complex([real[, imag]]) Index: Doc/library/compileall.rst =================================================================== --- Doc/library/compileall.rst (Revision 86724) +++ Doc/library/compileall.rst (Arbeitskopie) @@ -58,7 +58,7 @@ Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. The *maxlevels* parameter is used to limit the depth of @@ -76,15 +76,24 @@ If *legacy* is true, old-style ``.pyc`` file path names are written, otherwise (the default), :pep:`3147`-style path names are written. + *optimize* specifies the optimization level for the compiler. It is passed to + the built-in :func:`compile` function. -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, legacy=False) + .. versionchanged:: 3.2 + Added the *optimize* parameter. + +.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, legacy=False, optimize=-1) + Byte-compile all the :file:`.py` files found along ``sys.path``. If *skip_curdir* is true (the default), the current directory is not included in - the search. The *maxlevels* parameter defaults to ``0``, and the *force* - and *legacy* parameters default to ``False``. All are - passed to the :func:`compile_dir` function. + the search. All other parameters are passed to the :func:`compile_dir` + function. + .. versionchanged:: 3.2 + Added the *optimize* parameter. + + To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: Index: Lib/py_compile.py =================================================================== --- Lib/py_compile.py (Revision 86724) +++ Lib/py_compile.py (Arbeitskopie) @@ -72,7 +72,7 @@ (x >> 16) & 0xff, (x >> 24) & 0xff])) -def compile(file, cfile=None, dfile=None, doraise=False): +def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): """Byte-compile one Python source file to Python bytecode. :param file: The source file name. @@ -86,6 +86,10 @@ will be printed, and the function will return to the caller. If an exception occurs and this flag is set to True, a PyCompileError exception will be raised. + :param optimize: The optimization level for the compiler. Valid values + are -1, 0, 1 and 2. A value of -1 means to use the optimization + level of the current interpreter, as given by -O command line options. + :return: Path to the resulting byte compiled file. Note that it isn't necessary to byte-compile Python modules for @@ -111,7 +115,8 @@ timestamp = int(os.stat(file).st_mtime) codestring = f.read() try: - codeobject = builtins.compile(codestring, dfile or file,'exec') + codeobject = builtins.compile(codestring, dfile or file, 'exec', + optimize=optimize) except Exception as err: py_exc = PyCompileError(err.__class__, err, dfile or file) if doraise: @@ -120,7 +125,10 @@ sys.stderr.write(py_exc.msg + '\n') return if cfile is None: - cfile = imp.cache_from_source(file) + if optimize >= 0: + cfile = imp.cache_from_source(file, debug_override=not optimize) + else: + cfile = imp.cache_from_source(file) try: os.makedirs(os.path.dirname(cfile)) except OSError as error: Index: Lib/compileall.py =================================================================== --- Lib/compileall.py (Revision 86724) +++ Lib/compileall.py (Arbeitskopie) @@ -19,8 +19,8 @@ __all__ = ["compile_dir","compile_file","compile_path"] -def compile_dir(dir, maxlevels=10, ddir=None, - force=False, rx=None, quiet=False, legacy=False): +def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, + quiet=False, legacy=False, optimize=-1): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -32,6 +32,7 @@ force: if True, force compilation, even if timestamps are up-to-date quiet: if True, be quiet during compilation legacy: if True, produce legacy pyc paths instead of PEP 3147 paths + optimize: optimization level or -1 for level of the interpreter """ if not quiet: print('Listing', dir, '...') @@ -51,7 +52,8 @@ else: dfile = None if not os.path.isdir(fullname): - if not compile_file(fullname, ddir, force, rx, quiet, legacy): + if not compile_file(fullname, ddir, force, rx, quiet, + legacy, optimize): success = 0 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): @@ -61,7 +63,7 @@ return success def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False, - legacy=False): + legacy=False, optimize=-1): """Byte-compile file. fullname: the file to byte-compile ddir: if given, purported directory name (this is the @@ -69,6 +71,7 @@ force: if True, force compilation, even if timestamps are up-to-date quiet: if True, be quiet during compilation legacy: if True, produce legacy pyc paths instead of PEP 3147 paths + optimize: optimization level or -1 for level of the interpreter """ success = 1 name = os.path.basename(fullname) @@ -84,7 +87,11 @@ if legacy: cfile = fullname + ('c' if __debug__ else 'o') else: - cfile = imp.cache_from_source(fullname) + if optimize >= 0: + cfile = imp.cache_from_source(fullname, + debug_override=not optimize) + else: + cfile = imp.cache_from_source(fullname) cache_dir = os.path.dirname(cfile) head, tail = name[:-3], name[-3:] if tail == '.py': @@ -101,7 +108,8 @@ if not quiet: print('Compiling', fullname, '...') try: - ok = py_compile.compile(fullname, cfile, dfile, True) + ok = py_compile.compile(fullname, cfile, dfile, True, + optimize=optimize) except py_compile.PyCompileError as err: if quiet: print('*** Error compiling', fullname, '...') @@ -126,7 +134,7 @@ return success def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, - legacy=False): + legacy=False, optimize=-1): """Byte-compile all module on sys.path. Arguments (all optional): @@ -136,6 +144,7 @@ force: as for compile_dir() (default False) quiet: as for compile_dir() (default False) legacy: as for compile_dir() (default False) + optimize: as for compile_dir() (default -1) """ success = 1 for dir in sys.path: @@ -144,7 +153,7 @@ else: success = success and compile_dir(dir, maxlevels, None, force, quiet=quiet, - legacy=legacy) + legacy=legacy, optimize=optimize) return success Index: Lib/test/test_compileall.py =================================================================== --- Lib/test/test_compileall.py (Revision 86724) +++ Lib/test/test_compileall.py (Arbeitskopie) @@ -88,7 +88,16 @@ compileall.compile_file(data_file) self.assertFalse(os.path.exists(os.path.join(data_dir, '__pycache__'))) + def test_optimize(self): + # make sure compiling with different optimization settings than the + # interpreter's creates the correct file names + optimize = 1 if __debug__ else 0 + compileall.compile_dir(self.directory, quiet=True, optimize=optimize) + cached = imp.cache_from_source(self.source_path, + debug_override=not optimize) + self.assertTrue(os.path.isfile(cached)) + class EncodingTest(unittest.TestCase): """Issue 6716: compileall should escape source code when printing errors to stdout.""" Index: Lib/test/test_builtin.py =================================================================== --- Lib/test/test_builtin.py (Revision 86724) +++ Lib/test/test_builtin.py (Arbeitskopie) @@ -6,6 +6,7 @@ import warnings import collections import io +import ast import types import builtins import random @@ -268,6 +269,34 @@ self.assertRaises(TypeError, compile, chr(0), 'f', 'exec') self.assertRaises(ValueError, compile, str('a = 1'), 'f', 'bad') + # test the optimize argument + + codestr = '''def f(): + """doc""" + try: + assert False + except AssertionError: + return (True, f.__doc__) + else: + return (False, f.__doc__) + ''' + def f(): """doc""" + values = [(-1, __debug__, f.__doc__), + (0, True, 'doc'), + (1, False, 'doc'), + (2, False, None)] + for optval, debugval, docstring in values: + # test both direct compilation and compilation via AST + codeobjs = [] + codeobjs.append(compile(codestr, "", "exec", optimize=optval)) + tree = ast.parse(codestr) + codeobjs.append(compile(tree, "", "exec", optimize=optval)) + for code in codeobjs: + ns = {} + exec(code, ns) + rv = ns['f']() + self.assertEqual(rv, (debugval, docstring)) + def test_delattr(self): sys.spam = 1 delattr(sys, 'spam')