diff -r 9c929b9e0a2a Doc/c-api/veryhigh.rst --- a/Doc/c-api/veryhigh.rst Thu Nov 07 23:12:23 2013 +0100 +++ b/Doc/c-api/veryhigh.rst Thu Nov 07 23:42:21 2013 +0100 @@ -62,12 +62,19 @@ the same library that the Python runtime .. c:function:: int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) + Similar to :c:func:`PyRun_AnyFileObject`, but *filename* is an optional byte + string decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). If *filename* is *NULL*, this function + uses ``"???"`` as the filename. + +.. c:function:: int PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, PyCompilerFlags *flags) + If *fp* refers to a file associated with an interactive device (console or terminal input or Unix pseudo-terminal), return the value of :c:func:`PyRun_InteractiveLoop`, otherwise return the result of - :c:func:`PyRun_SimpleFile`. *filename* is decoded from the filesystem - encoding (:func:`sys.getfilesystemencoding`). If *filename* is *NULL*, this - function uses ``"???"`` as the filename. + :c:func:`PyRun_SimpleFile`. + + .. versionadded:: 3.4 .. c:function:: int PyRun_SimpleString(const char *command) @@ -103,11 +110,18 @@ the same library that the Python runtime .. c:function:: int PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) + Similar to :c:func:`PyRun_SimpleFileObject`, but *filename* is a byte + string decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). + +.. c:function:: int PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, PyCompilerFlags *flags) + Similar to :c:func:`PyRun_SimpleStringFlags`, but the Python source code is read from *fp* instead of an in-memory string. *filename* should be the name of - the file, it is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). If *closeit* is true, the file is - closed before PyRun_SimpleFileExFlags returns. + the file. If *closeit* is true, the file is closed before + PyRun_SimpleFileExFlags returns. + + .. versionadded:: 3.4 .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) @@ -118,10 +132,16 @@ the same library that the Python runtime .. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a byte + string decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). + + +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) + Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - filesystem encoding (:func:`sys.getfilesystemencoding`). + prompted using ``sys.ps1`` and ``sys.ps2``. Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code @@ -129,6 +149,8 @@ the same library that the Python runtime there was a parse error. (Note that :file:`errcode.h` is not included by :file:`Python.h`, so must be included specifically if needed.) + .. versionadded:: 3.4 + .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) @@ -138,10 +160,17 @@ the same library that the Python runtime .. c:function:: int PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) + Similar to :c:func:`PyRun_InteractiveLoopObject`, but *filename* is a byte + string decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). + +.. c:function:: int PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) + Read and execute statements from a file associated with an interactive device until EOF is reached. The user will be prompted using ``sys.ps1`` and - ``sys.ps2``. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF. + ``sys.ps2``. Returns ``0`` at EOF. + + .. versionadded:: 3.4 .. c:var:: int (*PyOS_InputHook)(void) @@ -247,12 +276,19 @@ the same library that the Python runtime .. c:function:: PyObject* PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) + Similar to :c:func:`PyRun_FileObject`, but *filename* is a byte string + decoded from the filesystem encoding (:func:`sys.getfilesystemencoding`). + + +.. c:function:: PyObject* PyRun_FileObject(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) + Similar to :c:func:`PyRun_StringFlags`, but the Python source code is read from - *fp* instead of an in-memory string. *filename* should be the name of the file, - it is decoded from the filesystem encoding (:func:`sys.getfilesystemencoding`). + *fp* instead of an in-memory string. *filename* should be the name of the file. If *closeit* is true, the file is closed before :c:func:`PyRun_FileExFlags` returns. + .. versionadded:: 3.4 + .. c:function:: PyObject* Py_CompileString(const char *str, const char *filename, int start) diff -r 9c929b9e0a2a Include/pythonrun.h --- a/Include/pythonrun.h Thu Nov 07 23:12:23 2013 +0100 +++ b/Include/pythonrun.h Thu Nov 07 23:42:21 2013 +0100 @@ -54,11 +54,21 @@ PyAPI_FUNC(int) PyRun_AnyFileExFlags( const char *filename, /* decoded from the filesystem encoding */ int closeit, PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_AnyFileObject( + FILE *fp, + PyObject *filename, + int closeit, + PyCompilerFlags *flags); PyAPI_FUNC(int) PyRun_SimpleFileExFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ int closeit, PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_SimpleFileObject( + FILE *fp, + PyObject *filename, + int closeit, + PyCompilerFlags *flags); PyAPI_FUNC(int) PyRun_InteractiveOneFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ @@ -71,6 +81,10 @@ PyAPI_FUNC(int) PyRun_InteractiveLoopFla FILE *fp, const char *filename, /* decoded from the filesystem encoding */ PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_InteractiveLoopObject( + FILE *fp, + PyObject *filename, + PyCompilerFlags *flags); PyAPI_FUNC(struct _mod *) PyParser_ASTFromString( const char *s, @@ -132,6 +146,14 @@ PyAPI_FUNC(PyObject *) PyRun_FileExFlags PyObject *locals, int closeit, PyCompilerFlags *flags); +PyAPI_FUNC(PyObject *) PyRun_FileObject( + FILE *fp, + PyObject *filename, + int start, + PyObject *globals, + PyObject *locals, + int closeit, + PyCompilerFlags *flags); #endif #ifdef Py_LIMITED_API diff -r 9c929b9e0a2a Lib/test/test_cmd_line_script.py --- a/Lib/test/test_cmd_line_script.py Thu Nov 07 23:12:23 2013 +0100 +++ b/Lib/test/test_cmd_line_script.py Thu Nov 07 23:42:21 2013 +0100 @@ -379,15 +379,29 @@ class CmdLineTest(unittest.TestCase): # Issue #16218 source = 'print(ascii(__file__))\n' - script_name = _make_test_script(os.curdir, name, source) - self.addCleanup(support.unlink, script_name) - rc, stdout, stderr = assert_python_ok(script_name) + with temp_dir() as script_dir: + script_name = _make_test_script(script_dir, name, source) + rc, stdout, stderr = assert_python_ok(script_name) self.assertEqual( ascii(script_name), stdout.rstrip().decode('ascii'), 'stdout=%r stderr=%r' % (stdout, stderr)) self.assertEqual(0, rc) + def test_unencodable_filename(self): + filename = support.TESTFN_UNENCODABLE + if filename is None: + self.skipTest("need an unencodable filename") + + with temp_dir() as script_dir: + source = 'print("undecodable filename")' + script_name = _make_test_script(script_dir, filename, source) + rc, stdout, stderr = assert_python_ok(script_name) + + self.assertEqual( + stdout.rstrip() + stderr.rstrip(), + b"undecodable filename") + def test_main(): support.run_unittest(CmdLineTest) diff -r 9c929b9e0a2a Modules/main.c --- a/Modules/main.c Thu Nov 07 23:12:23 2013 +0100 +++ b/Modules/main.c Thu Nov 07 23:42:21 2013 +0100 @@ -305,10 +305,9 @@ error: } static int -run_file(FILE *fp, const wchar_t *filename, PyCompilerFlags *p_cf) +run_file(FILE *fp, const wchar_t *filename_wstr, PyCompilerFlags *p_cf) { - PyObject *unicode, *bytes = NULL; - char *filename_str; + PyObject *filename; int run; /* call pending calls like signal handlers (SIGINT) */ @@ -317,24 +316,17 @@ run_file(FILE *fp, const wchar_t *filena return 1; } - if (filename) { - unicode = PyUnicode_FromWideChar(filename, wcslen(filename)); - if (unicode != NULL) { - bytes = PyUnicode_EncodeFSDefault(unicode); - Py_DECREF(unicode); - } - if (bytes != NULL) - filename_str = PyBytes_AsString(bytes); - else { - PyErr_Clear(); - filename_str = ""; - } + if (filename_wstr != NULL) + filename = PyUnicode_FromWideChar(filename_wstr, wcslen(filename_wstr)); + else + filename = PyUnicode_FromString(""); + if (filename == NULL) { + PyErr_Print(); + return 1; } - else - filename_str = ""; - run = PyRun_AnyFileExFlags(fp, filename_str, filename != NULL, p_cf); - Py_XDECREF(bytes); + run = PyRun_AnyFileObject(fp, filename, filename_wstr != NULL, p_cf); + Py_DECREF(filename); return run != 0; } diff -r 9c929b9e0a2a Python/pythonrun.c --- a/Python/pythonrun.c Thu Nov 07 23:12:23 2013 +0100 +++ b/Python/pythonrun.c Thu Nov 07 23:42:21 2013 +0100 @@ -87,8 +87,7 @@ static int initstdio(void); static void flush_io(void); static PyObject *run_mod(mod_ty, PyObject *, PyObject *, PyObject *, PyCompilerFlags *, PyArena *); -static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *, - PyCompilerFlags *); +static PyObject *run_pyc_file(FILE *, PyObject *, PyCompilerFlags *); static void err_input(perrdetail *); static void err_free(perrdetail *); static void initsigs(void); @@ -102,6 +101,8 @@ extern int _PyLong_Init(void); extern void PyLong_Fini(void); extern int _PyFaulthandler_Init(void); extern void _PyFaulthandler_Fini(void); +static int _Py_FdIsInteractiveObject(FILE *fp, PyObject *filename); + #ifdef WITH_THREAD extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); @@ -1261,33 +1262,47 @@ initstdio(void) /* Parse input from a file and execute it */ int -PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, - PyCompilerFlags *flags) +PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, + PyCompilerFlags *flags) { - if (filename == NULL) - filename = "???"; - if (Py_FdIsInteractive(fp, filename)) { - int err = PyRun_InteractiveLoopFlags(fp, filename, flags); + assert(filename != NULL); + if (_Py_FdIsInteractiveObject(fp, filename)) { + int err = PyRun_InteractiveLoopObject(fp, filename, flags); if (closeit) fclose(fp); return err; } else - return PyRun_SimpleFileExFlags(fp, filename, closeit, flags); + return PyRun_SimpleFileObject(fp, filename, closeit, flags); } int -PyRun_InteractiveLoopFlags(FILE *fp, const char *filename_str, PyCompilerFlags *flags) +PyRun_AnyFileExFlags(FILE *fp, const char *filename_str, int closeit, + PyCompilerFlags *flags) { - PyObject *filename, *v; - int ret, err; - PyCompilerFlags local_flags; + PyObject *filename; + int res; + + if (filename_str == NULL) + filename_str = "???"; filename = PyUnicode_DecodeFSDefault(filename_str); if (filename == NULL) { PyErr_Print(); return -1; } + res = PyRun_AnyFileObject(fp, filename, closeit, flags); + Py_DECREF(filename); + return res; +} + +int +PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, + PyCompilerFlags *flags) +{ + PyObject *v; + int ret, err; + PyCompilerFlags local_flags; if (flags == NULL) { flags = &local_flags; @@ -1316,8 +1331,27 @@ PyRun_InteractiveLoopFlags(FILE *fp, con break; */ } + return err; +} + +int +PyRun_InteractiveLoopFlags(FILE *fp, const char *filename_str, + PyCompilerFlags *flags) +{ + PyObject *filename; + int res; + + if (filename_str == NULL) + filename_str = "???"; + + filename = PyUnicode_DecodeFSDefault(filename_str); + if (filename == NULL) { + PyErr_Print(); + return -1; + } + res = PyRun_InteractiveLoopObject(fp, filename, flags); Py_DECREF(filename); - return err; + return res; } /* compute parser flags based on compiler flags */ @@ -1453,14 +1487,58 @@ PyRun_InteractiveOneFlags(FILE *fp, cons return res; } +/* filename.endswith(('.pyc', '.pyc')) */ +static int +filename_endswith_pyc_pyo(PyObject *filename) +{ + void* data; + int kind; + Py_UCS4 ch; + Py_ssize_t len; + + len = PyUnicode_GetLength(filename); + if (len <= 4) + return 0; + + data = PyUnicode_DATA(filename); + kind = PyUnicode_KIND(filename); + + ch = PyUnicode_READ(kind, data, len-1); + if ((ch != 'c' && ch != 'o')) + return 0; + + return (PyUnicode_READ(kind, data, len-2) == 'y' + && PyUnicode_READ(kind, data, len-3) == 'p' + && PyUnicode_READ(kind, data, len-4) == '.'); +} +static int +filename_endswith_pyo(PyObject *filename) +{ + void* data; + int kind; + Py_ssize_t len; + + len = PyUnicode_GetLength(filename); + if (len <= 4) + return 0; + + data = PyUnicode_DATA(filename); + kind = PyUnicode_KIND(filename); + + return (PyUnicode_READ(kind, data, len-1) == 'o' + && PyUnicode_READ(kind, data, len-2) == 'y' + && PyUnicode_READ(kind, data, len-3) == 'p' + && PyUnicode_READ(kind, data, len-4) == '.'); +} + /* Check whether a file maybe a pyc file: Look at the extension, the file type, and, if we may close it, at the first few bytes. */ static int -maybe_pyc_file(FILE *fp, const char* filename, const char* ext, int closeit) +maybe_pyc_file(FILE *fp, PyObject *filename, int closeit) { - if (strcmp(ext, ".pyc") == 0 || strcmp(ext, ".pyo") == 0) + if (filename_endswith_pyc_pyo(filename)) return 1; /* Only look into the file if we are allowed to close it, since @@ -1495,25 +1573,21 @@ maybe_pyc_file(FILE *fp, const char* fil } static int -set_main_loader(PyObject *d, const char *filename, const char *loader_name) +set_main_loader(PyObject *d, PyObject *filename, const char *loader_name) { PyInterpreterState *interp; PyThreadState *tstate; - PyObject *filename_obj, *loader_type, *loader; + PyObject *loader_type, *loader; int result = 0; - filename_obj = PyUnicode_DecodeFSDefault(filename); - if (filename_obj == NULL) - return -1; /* Get current thread state and interpreter pointer */ tstate = PyThreadState_GET(); interp = tstate->interp; loader_type = PyObject_GetAttrString(interp->importlib, loader_name); if (loader_type == NULL) { - Py_DECREF(filename_obj); return -1; } - loader = PyObject_CallFunction(loader_type, "sN", "__main__", filename_obj); + loader = PyObject_CallFunction(loader_type, "sO", "__main__", filename); Py_DECREF(loader_type); if (loader == NULL) { return -1; @@ -1526,13 +1600,11 @@ set_main_loader(PyObject *d, const char } int -PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, - PyCompilerFlags *flags) +PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, + PyCompilerFlags *flags) { PyObject *m, *d, *v; - const char *ext; int set_file_name = 0, ret = -1; - size_t len; m = PyImport_AddModule("__main__"); if (m == NULL) @@ -1540,34 +1612,23 @@ PyRun_SimpleFileExFlags(FILE *fp, const Py_INCREF(m); d = PyModule_GetDict(m); if (PyDict_GetItemString(d, "__file__") == NULL) { - PyObject *f; - f = PyUnicode_DecodeFSDefault(filename); - if (f == NULL) + if (PyDict_SetItemString(d, "__file__", filename) < 0) goto done; - if (PyDict_SetItemString(d, "__file__", f) < 0) { - Py_DECREF(f); + if (PyDict_SetItemString(d, "__cached__", Py_None) < 0) goto done; - } - if (PyDict_SetItemString(d, "__cached__", Py_None) < 0) { - Py_DECREF(f); - goto done; - } set_file_name = 1; - Py_DECREF(f); } - len = strlen(filename); - ext = filename + len - (len > 4 ? 4 : 0); - if (maybe_pyc_file(fp, filename, ext, closeit)) { + if (maybe_pyc_file(fp, filename, closeit)) { FILE *pyc_fp; /* Try to run a pyc file. First, re-open in binary */ if (closeit) fclose(fp); - if ((pyc_fp = _Py_fopen(filename, "rb")) == NULL) { + if ((pyc_fp = _Py_fopen_obj(filename, "rb")) == NULL) { fprintf(stderr, "python: Can't reopen .pyc file\n"); goto done; } /* Turn on optimization if a .pyo file is given */ - if (strcmp(ext, ".pyo") == 0) + if (filename_endswith_pyo(filename)) Py_OptimizeFlag = 1; if (set_main_loader(d, filename, "SourcelessFileLoader") < 0) { @@ -1576,18 +1637,18 @@ PyRun_SimpleFileExFlags(FILE *fp, const fclose(pyc_fp); goto done; } - v = run_pyc_file(pyc_fp, filename, d, d, flags); + v = run_pyc_file(pyc_fp, d, flags); fclose(pyc_fp); } else { /* When running from stdin, leave __main__.__loader__ alone */ - if (strcmp(filename, "") != 0 && + if (PyUnicode_CompareWithASCIIString(filename, "") != 0 && set_main_loader(d, filename, "SourceFileLoader") < 0) { fprintf(stderr, "python: failed to set __main__.__loader__\n"); ret = -1; goto done; } - v = PyRun_FileExFlags(fp, filename, Py_file_input, d, d, - closeit, flags); + v = PyRun_FileObject(fp, filename, Py_file_input, d, d, + closeit, flags); } flush_io(); if (v == NULL) { @@ -1604,6 +1665,21 @@ PyRun_SimpleFileExFlags(FILE *fp, const } int +PyRun_SimpleFileExFlags(FILE *fp, const char *filename_str, int closeit, + PyCompilerFlags *flags) +{ + PyObject *filename; + int res; + + filename = PyUnicode_DecodeFSDefault(filename_str); + if (filename == NULL) + return -1; + res = PyRun_SimpleFileObject(fp, filename, closeit, flags); + Py_DECREF(filename); + return res; +} + +int PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) { PyObject *m, *d, *v; @@ -2083,17 +2159,12 @@ PyRun_StringFlags(const char *str, int s } PyObject * -PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globals, - PyObject *locals, int closeit, PyCompilerFlags *flags) +PyRun_FileObject(FILE *fp, PyObject *filename, int start, PyObject *globals, + PyObject *locals, int closeit, PyCompilerFlags *flags) { PyObject *ret = NULL; mod_ty mod; PyArena *arena = NULL; - PyObject *filename; - - filename = PyUnicode_DecodeFSDefault(filename_str); - if (filename == NULL) - goto exit; arena = PyArena_New(); if (arena == NULL) @@ -2109,12 +2180,27 @@ PyRun_FileExFlags(FILE *fp, const char * ret = run_mod(mod, filename, globals, locals, flags, arena); exit: - Py_XDECREF(filename); if (arena != NULL) PyArena_Free(arena); return ret; } +PyObject * +PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, + PyObject *globals, PyObject *locals, int closeit, + PyCompilerFlags *flags) +{ + PyObject *filename, *res; + + filename = PyUnicode_DecodeFSDefault(filename_str); + if (filename == NULL) + return NULL; + res = PyRun_FileObject(fp, filename, start, globals, + locals, closeit, flags); + Py_DECREF(filename); + return res; +} + static void flush_io(void) { @@ -2160,8 +2246,7 @@ run_mod(mod_ty mod, PyObject *filename, } static PyObject * -run_pyc_file(FILE *fp, const char *filename, PyObject *globals, - PyObject *locals, PyCompilerFlags *flags) +run_pyc_file(FILE *fp, PyObject *globals, PyCompilerFlags *flags) { PyCodeObject *co; PyObject *v; @@ -2185,7 +2270,7 @@ run_pyc_file(FILE *fp, const char *filen return NULL; } co = (PyCodeObject *)v; - v = PyEval_EvalCode((PyObject*)co, globals, locals); + v = PyEval_EvalCode((PyObject*)co, globals, globals); if (v && flags) flags->cf_flags |= (co->co_flags & PyCF_MASK); Py_DECREF(co); @@ -2740,6 +2825,18 @@ Py_FdIsInteractive(FILE *fp, const char (strcmp(filename, "???") == 0); } +static int +_Py_FdIsInteractiveObject(FILE *fp, PyObject *filename) +{ + if (isatty((int)fileno(fp))) + return 1; + if (!Py_InteractiveFlag) + return 0; + return (filename == NULL) || + (PyUnicode_CompareWithASCIIString(filename, "") == 0) || + (PyUnicode_CompareWithASCIIString(filename, "???") == 0); +} + #if defined(USE_STACKCHECK) #if defined(WIN32) && defined(_MSC_VER)