diff -r 37975d542715 Include/Python.h --- a/Include/Python.h Sat Jun 04 00:53:32 2016 +0300 +++ b/Include/Python.h Fri Jun 03 16:52:14 2016 -0700 @@ -112,6 +112,7 @@ #include "pyarena.h" #include "modsupport.h" +#include "compile.h" #include "pythonrun.h" #include "pylifecycle.h" #include "ceval.h" @@ -122,7 +123,6 @@ #include "abstract.h" #include "bltinmodule.h" -#include "compile.h" #include "eval.h" #include "pyctype.h" diff -r 37975d542715 Include/compile.h --- a/Include/compile.h Sat Jun 04 00:53:32 2016 +0300 +++ b/Include/compile.h Fri Jun 03 16:52:14 2016 -0700 @@ -11,6 +11,23 @@ /* Public interface */ struct _node; /* Declare the existence of this type */ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); +/* XXX (ncoghlan): Unprefixed type name in a public API! */ + +#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \ + CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \ + CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ + CO_FUTURE_GENERATOR_STOP) +#define PyCF_MASK_OBSOLETE (CO_NESTED) +#define PyCF_SOURCE_IS_UTF8 0x0100 +#define PyCF_DONT_IMPLY_DEDENT 0x0200 +#define PyCF_ONLY_AST 0x0400 +#define PyCF_IGNORE_COOKIE 0x0800 + +#ifndef Py_LIMITED_API +typedef struct { + int cf_flags; /* bitmask of CO_xxx flags relevant to future */ +} PyCompilerFlags; +#endif /* Future feature support */ diff -r 37975d542715 Include/pylifecycle.h --- a/Include/pylifecycle.h Sat Jun 04 00:53:32 2016 +0300 +++ b/Include/pylifecycle.h Fri Jun 03 16:52:14 2016 -0700 @@ -19,8 +19,15 @@ */ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, const char *errors); + +/* PEP 432 Multi-phase initialization API (Private while provisional!) */ +PyAPI_FUNC(void) _Py_InitializeCore(const _PyCoreConfig *); +PyAPI_FUNC(int) _Py_IsCoreInitialized(void); +PyAPI_FUNC(int) _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *); +PyAPI_FUNC(int) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *); #endif +/* Initialization and finalization */ PyAPI_FUNC(void) Py_Initialize(void); PyAPI_FUNC(void) Py_InitializeEx(int); #ifndef Py_LIMITED_API @@ -29,10 +36,11 @@ PyAPI_FUNC(void) Py_Finalize(void); PyAPI_FUNC(int) Py_FinalizeEx(void); PyAPI_FUNC(int) Py_IsInitialized(void); + +/* Subinterpreter support */ PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); - /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level * exit functions. */ @@ -77,14 +85,15 @@ /* Internal -- various one-time initializations */ #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void); -PyAPI_FUNC(PyObject *) _PySys_Init(void); +PyAPI_FUNC(PyObject *) _PySys_BeginInit(void); +PyAPI_FUNC(int) _PySys_EndInit(PyObject *); PyAPI_FUNC(void) _PyImport_Init(void); PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod); PyAPI_FUNC(void) _PyImportHooks_Init(void); PyAPI_FUNC(int) _PyFrame_Init(void); PyAPI_FUNC(int) _PyFloat_Init(void); PyAPI_FUNC(int) PyByteArray_Init(void); -PyAPI_FUNC(void) _PyRandom_Init(void); +PyAPI_FUNC(void) _Py_HashRandomization_Init(_PyCoreConfig *core_config); #endif /* Various internal finalizers */ @@ -106,7 +115,7 @@ PyAPI_FUNC(void) _PyGC_Fini(void); PyAPI_FUNC(void) PySlice_Fini(void); PyAPI_FUNC(void) _PyType_Fini(void); -PyAPI_FUNC(void) _PyRandom_Fini(void); +PyAPI_FUNC(void) _Py_HashRandomization_Fini(void); PyAPI_DATA(PyThreadState *) _Py_Finalizing; #endif diff -r 37975d542715 Include/pystate.h --- a/Include/pystate.h Sat Jun 04 00:53:32 2016 +0300 +++ b/Include/pystate.h Fri Jun 03 16:52:14 2016 -0700 @@ -16,6 +16,30 @@ #ifdef Py_LIMITED_API typedef struct _is PyInterpreterState; #else + +/* Interpreter configuration settings */ + +typedef struct { + int ignore_environment; + int use_hash_seed; + unsigned long hash_seed; + int _disable_importlib; /* Needed by freeze_importlib */ +} _PyCoreConfig; + +#define _PyCoreConfig_INIT {0, -1, 0, 0} + +/* Placeholders while working on the new configuration API + * + * See PEP 432 for final anticipated contents + * + * For the moment, just handle the args to _Py_InitializeEx + */ +typedef struct { + int install_signal_handlers; +} _PyMainInterpreterConfig; + +#define _PyMainInterpreterConfig_INIT {-1} + typedef struct _is { struct _is *next; @@ -33,6 +57,8 @@ int codecs_initialized; int fscodec_initialized; + _PyCoreConfig core_config; + _PyMainInterpreterConfig config; #ifdef HAVE_DLOPEN int dlopenflags; #endif @@ -273,6 +299,7 @@ Don't use unless you know what you are doing! */ #ifndef Py_LIMITED_API PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Head(void); +PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Next(PyInterpreterState *); PyAPI_FUNC(PyThreadState *) PyInterpreterState_ThreadHead(PyInterpreterState *); PyAPI_FUNC(PyThreadState *) PyThreadState_Next(PyThreadState *); diff -r 37975d542715 Include/pythonrun.h --- a/Include/pythonrun.h Sat Jun 04 00:53:32 2016 +0300 +++ b/Include/pythonrun.h Fri Jun 03 16:52:14 2016 -0700 @@ -7,22 +7,6 @@ extern "C" { #endif -#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \ - CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \ - CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ - CO_FUTURE_GENERATOR_STOP) -#define PyCF_MASK_OBSOLETE (CO_NESTED) -#define PyCF_SOURCE_IS_UTF8 0x0100 -#define PyCF_DONT_IMPLY_DEDENT 0x0200 -#define PyCF_ONLY_AST 0x0400 -#define PyCF_IGNORE_COOKIE 0x0800 - -#ifndef Py_LIMITED_API -typedef struct { - int cf_flags; /* bitmask of CO_xxx flags relevant to future */ -} PyCompilerFlags; -#endif - #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *); PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *); diff -r 37975d542715 Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py Sat Jun 04 00:53:32 2016 +0300 +++ b/Lib/importlib/_bootstrap.py Fri Jun 03 16:52:14 2016 -0700 @@ -1152,12 +1152,14 @@ def _install(sys_module, _imp_module): - """Install importlib as the implementation of import.""" + """Install importers for builtin and frozen modules""" _setup(sys_module, _imp_module) sys.meta_path.append(BuiltinImporter) sys.meta_path.append(FrozenImporter) +def _install_external_importers(): + """Install importers that require external filesystem access""" global _bootstrap_external import _frozen_importlib_external _bootstrap_external = _frozen_importlib_external diff -r 37975d542715 Lib/test/coding20731.py --- a/Lib/test/coding20731.py Sat Jun 04 00:53:32 2016 +0300 +++ b/Lib/test/coding20731.py Fri Jun 03 16:52:14 2016 -0700 @@ -1,4 +1,4 @@ -#coding:latin1 - - - +#coding:latin1 + + + diff -r 37975d542715 Lib/test/test_cmd_line.py --- a/Lib/test/test_cmd_line.py Sat Jun 04 00:53:32 2016 +0300 +++ b/Lib/test/test_cmd_line.py Fri Jun 03 16:52:14 2016 -0700 @@ -485,8 +485,29 @@ cwd=tmpdir) self.assertEqual(out.strip(), b"ok") + +class IgnoreEnvironmentTest(unittest.TestCase): + + def run_ignoring_vars(self, predicate, **env_vars): + # Runs a subprocess with -E set, even though we're passing + # specific environment variables + # Logical inversion to match predicate check to a zero return + # code indicating success + code = "import sys; sys.stderr.write(str(sys.flags)); sys.exit(not ({}))".format(predicate) + return assert_python_ok('-E', '-c', code, **env_vars) + + def test_ignore_PYTHONPATH(self): + path = "should_be_ignored" + self.run_ignoring_vars("'{}' not in sys.path".format(path), + PYTHONPATH=path) + + def test_ignore_PYTHONHASHSEED(self): + self.run_ignoring_vars("sys.flags.hash_randomization == 1", + PYTHONHASHSEED="0") + + def test_main(): - test.support.run_unittest(CmdLineTest) + test.support.run_unittest(CmdLineTest, IgnoreEnvironmentTest) test.support.reap_children() if __name__ == "__main__": diff -r 37975d542715 Lib/test/test_site.py --- a/Lib/test/test_site.py Sat Jun 04 00:53:32 2016 +0300 +++ b/Lib/test/test_site.py Fri Jun 03 16:52:14 2016 -0700 @@ -170,6 +170,7 @@ @unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 " "user-site (site.ENABLE_USER_SITE)") def test_s_option(self): + # (ncoghlan) Change this to use script_helper... usersite = site.USER_SITE self.assertIn(usersite, sys.path) @@ -186,7 +187,7 @@ if usersite == site.getsitepackages()[0]: self.assertEqual(rc, 1) else: - self.assertEqual(rc, 0) + self.assertEqual(rc, 0, "User site still added to path with -s") env = os.environ.copy() env["PYTHONNOUSERSITE"] = "1" @@ -196,14 +197,16 @@ if usersite == site.getsitepackages()[0]: self.assertEqual(rc, 1) else: - self.assertEqual(rc, 0) + self.assertEqual(rc, 0, + "User site still added to path with PYTHONNOUSERSITE") env = os.environ.copy() env["PYTHONUSERBASE"] = "/tmp" rc = subprocess.call([sys.executable, '-c', 'import sys, site; sys.exit(site.USER_BASE.startswith("/tmp"))'], env=env) - self.assertEqual(rc, 1) + self.assertEqual(rc, 1, + "User base not set by PYTHONUSERBASE") def test_getuserbase(self): site.USER_BASE = None diff -r 37975d542715 Makefile.pre.in --- a/Makefile.pre.in Sat Jun 04 00:53:32 2016 +0300 +++ b/Makefile.pre.in Fri Jun 03 16:52:14 2016 -0700 @@ -220,6 +220,8 @@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) +# TODO: Be able to build this under Programs without it crashing when run +SYSPYTHON= system-python$(EXE) cross_compiling=@cross_compiling@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ @@ -372,6 +374,7 @@ Python/asdl.o \ Python/ast.o \ Python/bltinmodule.o \ + Python/bootstrap_hash.o \ Python/ceval.o \ Python/compile.o \ Python/codecs.o \ @@ -401,7 +404,6 @@ Python/pystate.o \ Python/pythonrun.o \ Python/pytime.o \ - Python/random.o \ Python/structmember.o \ Python/symtable.o \ Python/sysmodule.o \ @@ -482,7 +484,7 @@ # Default target all: build_all -build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config +build_all: $(BUILDPYTHON) $(SYSPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config # Compile a binary with profile guided optimization. profile-opt: @@ -570,6 +572,9 @@ $(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) $(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) +$(SYSPYTHON): Programs/system-python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) + $(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) + platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -1085,6 +1090,9 @@ fi; \ done +# TODO: Decide how to handle installation of $(SYSPYTHON) +# (A dedicated make target likely makes the most sense) + # Install the interpreter with $(VERSION) affixed # This goes into $(exec_prefix) altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ @@ -1267,7 +1275,7 @@ else \ $(INSTALL_DATA) $$i $(DESTDIR)$(LIBDEST); \ echo $(INSTALL_DATA) $$i $(LIBDEST); \ - fi; \ + fi; done @for d in $(LIBSUBDIRS); \ do \ diff -r 37975d542715 Modules/main.c --- a/Modules/main.c Sat Jun 04 00:53:32 2016 +0300 +++ b/Modules/main.c Fri Jun 03 16:52:14 2016 -0700 @@ -324,48 +324,44 @@ /* Main program */ -int -Py_Main(int argc, wchar_t **argv) +/*TODO: Add arg processing to PEP 432 as a new configuration setup API + */ +typedef struct { + wchar_t *filename; /* Trailing arg without -c or -m */ + wchar_t *command; /* -c argument */ + wchar_t *module; /* -m argument */ + PyObject *warning_options; /* -W options */ + PyObject *extra_options; /* -X options */ + int print_help; /* -h, -? options */ + int print_version; /* -V option */ + int bytes_warning; /* Py_BytesWarningFlag */ + int debug; /* Py_DebugFlag */ + int inspect; /* Py_InspectFlag */ + int interactive; /* Py_InteractiveFlag */ + int isolated; /* Py_IsolatedFlag */ + int optimization_level; /* Py_OptimizeFlag */ + int dont_write_bytecode; /* Py_DontWriteBytecodeFlag */ + int no_user_site_directory; /* Py_NoUserSiteDirectory */ + int no_site_import; /* Py_NoSiteFlag */ + int use_unbuffered_io; /* Py_UnbufferedStdioFlag */ + int verbosity; /* Py_VerboseFlag */ + int quiet_flag; /* Py_QuietFlag */ + int skip_first_line; /* -x option */ +} _Py_CommandLineDetails; + +#define _Py_CommandLineDetails_INIT \ + {NULL, NULL, NULL, NULL, NULL, \ + 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0, 0} + +static int +read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) { - int c; - int sts; + PyObject *warning_option = NULL; wchar_t *command = NULL; - wchar_t *filename = NULL; wchar_t *module = NULL; - FILE *fp = stdin; - char *p; -#ifdef MS_WINDOWS - wchar_t *wp; -#endif - int skipfirstline = 0; - int stdin_is_interactive = 0; - int help = 0; - int version = 0; - int saw_unbuffered_flag = 0; + char c; char *opt; - PyCompilerFlags cf; - PyObject *warning_option = NULL; - PyObject *warning_options = NULL; - - cf.cf_flags = 0; - - orig_argc = argc; /* For Py_GetArgcArgv() */ - orig_argv = argv; - - /* Hash randomization needed early for all string operations - (including -W and -X options). */ - _PyOS_opterr = 0; /* prevent printing the error in 1st pass */ - while ((c = _PyOS_GetOpt(argc, argv, PROGRAM_OPTS)) != EOF) { - if (c == 'm' || c == 'c') { - /* -c / -m is the last option: following arguments are - not interpreter options. */ - break; - } - if (c == 'E') { - Py_IgnoreEnvironmentFlag++; - break; - } - } opt = Py_GETENV("PYTHONMALLOC"); if (_PyMem_SetupAllocators(opt) < 0) { @@ -374,10 +370,6 @@ exit(1); } - Py_HashRandomizationFlag = 1; - _PyRandom_Init(); - - PySys_ResetWarnOptions(); _PyOS_ResetGetOpt(); while ((c = _PyOS_GetOpt(argc, argv, PROGRAM_OPTS)) != EOF) { @@ -395,6 +387,7 @@ wcscpy(command, _PyOS_optarg); command[len - 2] = '\n'; command[len - 1] = 0; + cmdline->command = command; break; } @@ -403,49 +396,49 @@ that look like options are left for the module to interpret. */ module = _PyOS_optarg; + cmdline->module = module; break; } switch (c) { case 'b': - Py_BytesWarningFlag++; + cmdline->bytes_warning++; break; case 'd': - Py_DebugFlag++; + cmdline->debug++; break; case 'i': - Py_InspectFlag++; - Py_InteractiveFlag++; + cmdline->inspect++; + cmdline->interactive++; break; case 'I': - Py_IsolatedFlag++; - Py_NoUserSiteDirectory++; - Py_IgnoreEnvironmentFlag++; + cmdline->isolated++; + cmdline->no_user_site_directory++; break; /* case 'J': reserved for Jython */ case 'O': - Py_OptimizeFlag++; + cmdline->optimization_level++; break; case 'B': - Py_DontWriteBytecodeFlag++; + cmdline->dont_write_bytecode++; break; case 's': - Py_NoUserSiteDirectory++; + cmdline->no_user_site_directory++; break; case 'S': - Py_NoSiteFlag++; + cmdline->no_site_import++; break; case 'E': - /* Already handled above */ + /* Handled prior to core initialization */ break; case 't': @@ -453,45 +446,45 @@ break; case 'u': - Py_UnbufferedStdioFlag = 1; - saw_unbuffered_flag = 1; + cmdline->use_unbuffered_io = 1; break; case 'v': - Py_VerboseFlag++; + cmdline->verbosity++; break; case 'x': - skipfirstline = 1; + cmdline->skip_first_line = 1; break; case 'h': case '?': - help++; + cmdline->print_help++; break; case 'V': - version++; + cmdline->print_version++; break; case 'W': - if (warning_options == NULL) - warning_options = PyList_New(0); - if (warning_options == NULL) + if (cmdline->warning_options == NULL) + cmdline->warning_options = PyList_New(0); + if (cmdline->warning_options == NULL) Py_FatalError("failure in handling of -W argument"); warning_option = PyUnicode_FromWideChar(_PyOS_optarg, -1); if (warning_option == NULL) Py_FatalError("failure in handling of -W argument"); - PyList_Append(warning_options, warning_option); + PyList_Append(cmdline->warning_options, warning_option); Py_DECREF(warning_option); break; case 'X': + /* TODO: Delay addition of X options to sys module */ PySys_AddXOption(_PyOS_optarg); break; case 'q': - Py_QuietFlag++; + cmdline->quiet_flag++; break; case 'R': @@ -501,30 +494,115 @@ /* This space reserved for other options */ default: - return usage(2, argv[0]); + return -1; /*NOTREACHED*/ } } - if (help) + if (command == NULL && module == NULL && _PyOS_optind < argc && + wcscmp(argv[_PyOS_optind], L"-") != 0) + { + cmdline->filename = argv[_PyOS_optind]; + } + return 0; +} + +static int +apply_command_line_and_environment(_Py_CommandLineDetails *cmdline) +{ + char *p; + Py_BytesWarningFlag = cmdline->bytes_warning; + Py_DebugFlag = cmdline->debug; + Py_InspectFlag = cmdline->inspect; + Py_InteractiveFlag = cmdline->interactive; + Py_IsolatedFlag = cmdline->isolated; + Py_OptimizeFlag = cmdline->optimization_level; + Py_DontWriteBytecodeFlag = cmdline->dont_write_bytecode; + Py_NoUserSiteDirectory = cmdline->no_user_site_directory; + Py_NoSiteFlag = cmdline->no_site_import; + Py_UnbufferedStdioFlag = cmdline->use_unbuffered_io; + Py_VerboseFlag = cmdline->verbosity; + Py_QuietFlag = cmdline->quiet_flag; + + if (!Py_InspectFlag && + (p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') { + Py_InspectFlag = 1; + cmdline->inspect = 1; + } + if (!cmdline->use_unbuffered_io && + (p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0') { + Py_UnbufferedStdioFlag = 1; + cmdline->use_unbuffered_io = 1; + } + + if (!Py_NoUserSiteDirectory && + (p = Py_GETENV("PYTHONNOUSERSITE")) && *p != '\0') { + Py_NoUserSiteDirectory = 1; + cmdline->no_user_site_directory = 1; + } + + /* TODO: Apply PYTHONWARNINGS & -W options to sys module here */ + /* TODO: Apply -X options to sys module here */ + return 0; +} + +int +Py_Main(int argc, wchar_t **argv) +{ + int c; + int sts; + FILE *fp = stdin; + char *p; +#ifdef MS_WINDOWS + wchar_t *wp; +#endif + int stdin_is_interactive = 0; + _Py_CommandLineDetails cmdline = _Py_CommandLineDetails_INIT; + _PyCoreConfig core_config = _PyCoreConfig_INIT; + PyCompilerFlags cf; + + cf.cf_flags = 0; + + orig_argc = argc; /* For Py_GetArgcArgv() */ + orig_argv = argv; + + /* Hash randomization needed early for all string operations + (including -W and -X options). */ + _PyOS_opterr = 0; /* prevent printing the error in 1st pass */ + while ((c = _PyOS_GetOpt(argc, argv, PROGRAM_OPTS)) != EOF) { + if (c == 'm' || c == 'c') { + /* -c / -m is the last option: following arguments are + not interpreter options. */ + break; + } + if (c == 'E' || c == 'I') { + core_config.ignore_environment++; + break; + } + } + + /* Initialize the core language runtime */ + Py_IgnoreEnvironmentFlag = core_config.ignore_environment; + core_config._disable_importlib = 0; + _Py_InitializeCore(&core_config); + + /* Reprocess the command line with the language runtime available */ + if (read_command_line(argc, argv, &cmdline)) { + return usage(2, argv[0]); + } + + if (cmdline.print_help) { return usage(0, argv[0]); + } - if (version) { + if (cmdline.print_version) { printf("Python %s\n", PY_VERSION); return 0; } - if (!Py_InspectFlag && - (p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') - Py_InspectFlag = 1; - if (!saw_unbuffered_flag && - (p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0') - Py_UnbufferedStdioFlag = 1; - - if (!Py_NoUserSiteDirectory && - (p = Py_GETENV("PYTHONNOUSERSITE")) && *p != '\0') - Py_NoUserSiteDirectory = 1; + PySys_ResetWarnOptions(); + apply_command_line_and_environment(&cmdline); #ifdef MS_WINDOWS if (!Py_IgnoreEnvironmentFlag && (wp = _wgetenv(L"PYTHONWARNINGS")) && @@ -577,18 +655,13 @@ PyMem_RawFree(buf); } #endif - if (warning_options != NULL) { + if (cmdline.warning_options != NULL) { Py_ssize_t i; - for (i = 0; i < PyList_GET_SIZE(warning_options); i++) { - PySys_AddWarnOptionUnicode(PyList_GET_ITEM(warning_options, i)); + for (i = 0; i < PyList_GET_SIZE(cmdline.warning_options); i++) { + PySys_AddWarnOptionUnicode(PyList_GET_ITEM(cmdline.warning_options, i)); } } - - if (command == NULL && module == NULL && _PyOS_optind < argc && - wcscmp(argv[_PyOS_optind], L"-") != 0) - { - filename = argv[_PyOS_optind]; - } + Py_CLEAR(cmdline.warning_options); stdin_is_interactive = Py_FdIsInteractive(stdin, (char *)0); @@ -675,25 +748,40 @@ #else Py_SetProgramName(argv[0]); #endif - Py_Initialize(); - Py_XDECREF(warning_options); + /* Replaces previous call to Py_Initialize() + * + * TODO: Move environment queries (etc) into Py_ReadConfig + */ + { + _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; + /* TODO: Moar config options! */ + config.install_signal_handlers = 1; + /* TODO: Print any exceptions raised by these operations */ + if (_Py_ReadMainInterpreterConfig(&config)) + Py_FatalError("Py_Main: Py_ReadMainInterpreterConfig failed"); + if (_Py_InitializeMainInterpreter(&config)) + Py_FatalError("Py_Main: Py_InitializeMainInterpreter failed"); + } + + /* TODO: Move this to _PyRun_PrepareMain */ if (!Py_QuietFlag && (Py_VerboseFlag || - (command == NULL && filename == NULL && - module == NULL && stdin_is_interactive))) { + (cmdline.command == NULL && cmdline.filename == NULL && + cmdline.module == NULL && stdin_is_interactive))) { fprintf(stderr, "Python %s on %s\n", Py_GetVersion(), Py_GetPlatform()); if (!Py_NoSiteFlag) fprintf(stderr, "%s\n", COPYRIGHT); } - if (command != NULL) { + /* TODO: Move this to _Py_InitializeMainInterpreter */ + if (cmdline.command != NULL) { /* Backup _PyOS_optind and force sys.argv[0] = '-c' */ _PyOS_optind--; argv[_PyOS_optind] = L"-c"; } - if (module != NULL) { + if (cmdline.module != NULL) { /* Backup _PyOS_optind and force sys.argv[0] = '-m'*/ _PyOS_optind--; argv[_PyOS_optind] = L"-m"; @@ -701,8 +789,10 @@ PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); - if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) && - isatty(fileno(stdin))) { + if ((Py_InspectFlag || (cmdline.command == NULL && + cmdline.filename == NULL && + cmdline.module == NULL)) && + isatty(fileno(stdin))) { PyObject *v; v = PyImport_ImportModule("readline"); if (v == NULL) @@ -711,15 +801,15 @@ Py_DECREF(v); } - if (command) { - sts = run_command(command, &cf); - PyMem_RawFree(command); - } else if (module) { - sts = (RunModule(module, 1) != 0); + if (cmdline.command) { + sts = run_command(cmdline.command, &cf); + PyMem_RawFree(cmdline.command); + } else if (cmdline.module) { + sts = (RunModule(cmdline.module, 1) != 0); } else { - if (filename == NULL && stdin_is_interactive) { + if (cmdline.filename == NULL && stdin_is_interactive) { Py_InspectFlag = 0; /* do exit on SystemExit */ RunStartupFile(&cf); RunInteractiveHook(); @@ -728,17 +818,17 @@ sts = -1; /* keep track of whether we've already run __main__ */ - if (filename != NULL) { - sts = RunMainFromImporter(filename); + if (cmdline.filename != NULL) { + sts = RunMainFromImporter(cmdline.filename); } - if (sts==-1 && filename!=NULL) { - fp = _Py_wfopen(filename, L"r"); + if (sts==-1 && cmdline.filename!=NULL) { + fp = _Py_wfopen(cmdline.filename, L"r"); if (fp == NULL) { char *cfilename_buffer; const char *cfilename; int err = errno; - cfilename_buffer = Py_EncodeLocale(filename, NULL); + cfilename_buffer = Py_EncodeLocale(cmdline.filename, NULL); if (cfilename_buffer != NULL) cfilename = cfilename_buffer; else @@ -749,7 +839,7 @@ PyMem_Free(cfilename_buffer); return 2; } - else if (skipfirstline) { + else if (cmdline.skip_first_line) { int ch; /* Push back first newline so line numbers remain the same */ @@ -766,7 +856,7 @@ S_ISDIR(sb.st_mode)) { fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", - argv[0], filename); + argv[0], cmdline.filename); fclose(fp); return 1; } @@ -774,7 +864,7 @@ } if (sts == -1) - sts = run_file(fp, filename, &cf); + sts = run_file(fp, cmdline.filename, &cf); } /* Check this environment variable at the end, to give programs the @@ -787,7 +877,7 @@ } if (Py_InspectFlag && stdin_is_interactive && - (filename != NULL || command != NULL || module != NULL)) { + (cmdline.filename != NULL || cmdline.command != NULL || cmdline.module != NULL)) { Py_InspectFlag = 0; RunInteractiveHook(); /* XXX */ diff -r 37975d542715 PCbuild/_freeze_importlib.vcxproj --- a/PCbuild/_freeze_importlib.vcxproj Sat Jun 04 00:53:32 2016 +0300 +++ b/PCbuild/_freeze_importlib.vcxproj Fri Jun 03 16:52:14 2016 -0700 @@ -1,4 +1,4 @@ - + diff -r 37975d542715 PCbuild/_freeze_importlib.vcxproj.filters --- a/PCbuild/_freeze_importlib.vcxproj.filters Sat Jun 04 00:53:32 2016 +0300 +++ b/PCbuild/_freeze_importlib.vcxproj.filters Fri Jun 03 16:52:14 2016 -0700 @@ -1,4 +1,4 @@ - + diff -r 37975d542715 PCbuild/python.vcxproj --- a/PCbuild/python.vcxproj Sat Jun 04 00:53:32 2016 +0300 +++ b/PCbuild/python.vcxproj Fri Jun 03 16:52:14 2016 -0700 @@ -1,4 +1,4 @@ - + diff -r 37975d542715 PCbuild/python.vcxproj.filters --- a/PCbuild/python.vcxproj.filters Sat Jun 04 00:53:32 2016 +0300 +++ b/PCbuild/python.vcxproj.filters Fri Jun 03 16:52:14 2016 -0700 @@ -1,4 +1,4 @@ - + diff -r 37975d542715 PCbuild/pythoncore.vcxproj --- a/PCbuild/pythoncore.vcxproj Sat Jun 04 00:53:32 2016 +0300 +++ b/PCbuild/pythoncore.vcxproj Fri Jun 03 16:52:14 2016 -0700 @@ -1,4 +1,4 @@ - + @@ -342,11 +342,11 @@ - + @@ -384,6 +384,7 @@ + diff -r 37975d542715 PCbuild/pythoncore.vcxproj.filters --- a/PCbuild/pythoncore.vcxproj.filters Sat Jun 04 00:53:32 2016 +0300 +++ b/PCbuild/pythoncore.vcxproj.filters Fri Jun 03 16:52:14 2016 -0700 @@ -1,4 +1,4 @@ - + @@ -821,6 +821,9 @@ Python + + Python + Python @@ -932,6 +935,9 @@ Python + + Python + Python @@ -947,7 +953,7 @@ Python - + Python diff -r 37975d542715 Programs/system-python.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Programs/system-python.c Fri Jun 03 16:52:14 2016 -0700 @@ -0,0 +1,142 @@ +/* Main program tailored to ignore environmental & per-user settings */ + +#include "Python.h" +#include + +#ifdef __FreeBSD__ +#include +#endif + +#define _SET_CONFIG_NONE(target) \ + do { \ + Py_INCREF(Py_None); \ + (target) = Py_None; \ + } while (0) + +#define _SET_CONFIG_TEXT(target, value) \ + do { \ + PyUnicodeObject *_config_tmp = (value); \ + Py_INCREF(_config_tmp); \ + (target) = _config_tmp; \ + } while (0) + +static int +isolated_main(int argc, wchar_t **argv) +{ + /* No customisation yet, just use Py_Main */ + return Py_Main(argc, argv); + + /* Desired end state: + _PyCoreConfig core_config = _PyCoreConfig_INIT; + _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; + PyObject *encoding = NULL; + PyObject *errors = NULL; + core_config.ignore_environment = 1; + core_config.use_hash_seed = 0; + _Py_InitializeCore(&core_config); + // TODO: Report exception details & return an error code from here on out + config.no_user_site = 1; + config.dont_write_bytecode = 1; + config.run_implicit_code = 0; + // TODO: Consider handling of prefix and exec_prefix + // TODO: Consider handling of pyvenv.cfg (or lack thereof) + // TODO?: Read encoding setting from /etc/locale.d? + encoding = PyUnicode_InternFromString("utf-8"); + if (encoding == NULL) { + Py_FatalError("system-python: failed to create encoding string"); + } + errors = PyUnicode_InternFromString("surrogateescape"); + if (errors == NULL) { + Py_FatalError("system-python: failed to create errors string"); + } + _SET_CONFIG_TEXT(config.stdin_encoding, encoding); + _SET_CONFIG_TEXT(config.stdin_errors, errors); + _SET_CONFIG_TEXT(config.stdout_encoding, encoding); + _SET_CONFIG_TEXT(config.stdout_errors, errors); + _SET_CONFIG_TEXT(config.stderr_encoding, encoding); + _SET_CONFIG_TEXT(config.stderr_errors, errors); + _SET_CONFIG_TEXT(config.fs_encoding, encoding); + if (_Py_ReadMainInterpreterConfig(&config)) { + Py_FatalError("system-python: Py_ReadMainInterpreterConfig failed"); + } + if (_Py_InitializeMainInterpreter(&config)) { + Py_FatalError("system-python: Py_InitializeMainInterpreter failed"); + } + if (_Py_RunPrepareMain()) { + Py_FatalError("system-python: Py_RunPrepareMain failed"); + } + if (_Py_RunExecMain()) { + Py_FatalError("system-python: Py_RunExecMain failed"); + } + */ + +} + +#ifdef MS_WINDOWS +int +wmain(int argc, wchar_t **argv) +{ + return isolated_main(argc, argv); +} +#else + +int +main(int argc, char **argv) +{ + wchar_t **argv_copy; + /* We need a second copy, as Python might modify the first one. */ + wchar_t **argv_copy2; + int i, res; + char *oldloc; +#ifdef __FreeBSD__ + fp_except_t m; +#endif + + argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); + argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); + if (!argv_copy || !argv_copy2) { + fprintf(stderr, "out of memory\n"); + return 1; + } + + /* 754 requires that FP exceptions run in "no stop" mode by default, + * and until C vendors implement C99's ways to control FP exceptions, + * Python requires non-stop mode. Alas, some platforms enable FP + * exceptions by default. Here we disable them. + */ +#ifdef __FreeBSD__ + m = fpgetmask(); + fpsetmask(m & ~FP_X_OFL); +#endif + + oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL)); + if (!oldloc) { + fprintf(stderr, "out of memory\n"); + return 1; + } + + setlocale(LC_ALL, ""); + for (i = 0; i < argc; i++) { + argv_copy[i] = Py_DecodeLocale(argv[i], NULL); + if (!argv_copy[i]) { + PyMem_RawFree(oldloc); + fprintf(stderr, "Fatal Python error: " + "unable to decode the command line argument #%i\n", + i + 1); + return 1; + } + argv_copy2[i] = argv_copy[i]; + } + argv_copy2[argc] = argv_copy[argc] = NULL; + + setlocale(LC_ALL, oldloc); + PyMem_RawFree(oldloc); + res = isolated_main(argc, argv_copy); + for (i = 0; i < argc; i++) { + PyMem_RawFree(argv_copy2[i]); + } + PyMem_RawFree(argv_copy); + PyMem_RawFree(argv_copy2); + return res; +} +#endif diff -r 37975d542715 Python/bootstrap_hash.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Python/bootstrap_hash.c Fri Jun 03 16:52:14 2016 -0700 @@ -0,0 +1,476 @@ +#include "Python.h" +#ifdef MS_WINDOWS +# include +#else +# include +# ifdef HAVE_SYS_STAT_H +# include +# endif +# ifdef HAVE_GETRANDOM +# include +# elif defined(HAVE_GETRANDOM_SYSCALL) +# include +# endif +#endif + +#ifdef Py_DEBUG +int _Py_HashSecret_Initialized = 0; +#else +static int _Py_HashSecret_Initialized = 0; +#endif + +#ifdef MS_WINDOWS +static HCRYPTPROV hCryptProv = 0; + +static int +win32_urandom_init(int raise) +{ + /* Acquire context */ + if (!CryptAcquireContext(&hCryptProv, NULL, NULL, + PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) + goto error; + + return 0; + +error: + if (raise) + PyErr_SetFromWindowsErr(0); + else + Py_FatalError("Failed to initialize Windows random API (CryptoGen)"); + return -1; +} + +/* Fill buffer with size pseudo-random bytes generated by the Windows CryptoGen + API. Return 0 on success, or raise an exception and return -1 on error. */ +static int +win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise) +{ + Py_ssize_t chunk; + + if (hCryptProv == 0) + { + if (win32_urandom_init(raise) == -1) + return -1; + } + + while (size > 0) + { + chunk = size > INT_MAX ? INT_MAX : size; + if (!CryptGenRandom(hCryptProv, (DWORD)chunk, buffer)) + { + /* CryptGenRandom() failed */ + if (raise) + PyErr_SetFromWindowsErr(0); + else + Py_FatalError("Failed to initialized the randomized hash " + "secret using CryptoGen)"); + return -1; + } + buffer += chunk; + size -= chunk; + } + return 0; +} + +#elif defined(HAVE_GETENTROPY) && !defined(sun) +#define PY_GETENTROPY 1 + +/* Fill buffer with size pseudo-random bytes generated by getentropy(). + Return 0 on success, or raise an exception and return -1 on error. + + If fatal is nonzero, call Py_FatalError() instead of raising an exception + on error. */ +static int +py_getentropy(unsigned char *buffer, Py_ssize_t size, int fatal) +{ + while (size > 0) { + Py_ssize_t len = Py_MIN(size, 256); + int res; + + if (!fatal) { + Py_BEGIN_ALLOW_THREADS + res = getentropy(buffer, len); + Py_END_ALLOW_THREADS + + if (res < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + } + else { + res = getentropy(buffer, len); + if (res < 0) + Py_FatalError("getentropy() failed"); + } + + buffer += len; + size -= len; + } + return 0; +} + +#else + +/* Issue #25003: Don' use getentropy() on Solaris (available since + * Solaris 11.3), it is blocking whereas os.urandom() should not block. */ +#if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL) +#define PY_GETRANDOM 1 + +static int +py_getrandom(void *buffer, Py_ssize_t size, int raise) +{ + /* Is getrandom() supported by the running kernel? + * Need Linux kernel 3.17 or newer, or Solaris 11.3 or newer */ + static int getrandom_works = 1; + /* Use non-blocking /dev/urandom device. On Linux at boot, the getrandom() + * syscall blocks until /dev/urandom is initialized with enough entropy. */ + const int flags = 0; + int n; + + if (!getrandom_works) + return 0; + + while (0 < size) { +#ifdef sun + /* Issue #26735: On Solaris, getrandom() is limited to returning up + to 1024 bytes */ + n = Py_MIN(size, 1024); +#else + n = size; +#endif + + errno = 0; +#ifdef HAVE_GETRANDOM + if (raise) { + Py_BEGIN_ALLOW_THREADS + n = getrandom(buffer, n, flags); + Py_END_ALLOW_THREADS + } + else { + n = getrandom(buffer, n, flags); + } +#else + /* On Linux, use the syscall() function because the GNU libc doesn't + * expose the Linux getrandom() syscall yet. See: + * https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */ + if (raise) { + Py_BEGIN_ALLOW_THREADS + n = syscall(SYS_getrandom, buffer, n, flags); + Py_END_ALLOW_THREADS + } + else { + n = syscall(SYS_getrandom, buffer, n, flags); + } +#endif + + if (n < 0) { + if (errno == ENOSYS) { + getrandom_works = 0; + return 0; + } + + if (errno == EINTR) { + if (PyErr_CheckSignals()) { + if (!raise) + Py_FatalError("getrandom() interrupted by a signal"); + return -1; + } + /* retry getrandom() */ + continue; + } + + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + else + Py_FatalError("getrandom() failed"); + return -1; + } + + buffer += n; + size -= n; + } + return 1; +} +#endif + +static struct { + int fd; + dev_t st_dev; + ino_t st_ino; +} urandom_cache = { -1 }; + + +/* Read size bytes from /dev/urandom into buffer. + Call Py_FatalError() on error. */ +static void +dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size) +{ + int fd; + Py_ssize_t n; + + assert (0 < size); + +#ifdef PY_GETRANDOM + if (py_getrandom(buffer, size, 0) == 1) + return; + /* getrandom() is not supported by the running kernel, fall back + * on reading /dev/urandom */ +#endif + + fd = _Py_open_noraise("/dev/urandom", O_RDONLY); + if (fd < 0) + Py_FatalError("Failed to open /dev/urandom"); + + while (0 < size) + { + do { + n = read(fd, buffer, (size_t)size); + } while (n < 0 && errno == EINTR); + if (n <= 0) + { + /* stop on error or if read(size) returned 0 */ + Py_FatalError("Failed to read bytes from /dev/urandom"); + break; + } + buffer += n; + size -= (Py_ssize_t)n; + } + close(fd); +} + +/* Read size bytes from /dev/urandom into buffer. + Return 0 on success, raise an exception and return -1 on error. */ +static int +dev_urandom_python(char *buffer, Py_ssize_t size) +{ + int fd; + Py_ssize_t n; + struct _Py_stat_struct st; +#ifdef PY_GETRANDOM + int res; +#endif + + if (size <= 0) + return 0; + +#ifdef PY_GETRANDOM + res = py_getrandom(buffer, size, 1); + if (res < 0) + return -1; + if (res == 1) + return 0; + /* getrandom() is not supported by the running kernel, fall back + * on reading /dev/urandom */ +#endif + + if (urandom_cache.fd >= 0) { + /* Does the fd point to the same thing as before? (issue #21207) */ + if (_Py_fstat_noraise(urandom_cache.fd, &st) + || st.st_dev != urandom_cache.st_dev + || st.st_ino != urandom_cache.st_ino) { + /* Something changed: forget the cached fd (but don't close it, + since it probably points to something important for some + third-party code). */ + urandom_cache.fd = -1; + } + } + if (urandom_cache.fd >= 0) + fd = urandom_cache.fd; + else { + fd = _Py_open("/dev/urandom", O_RDONLY); + if (fd < 0) { + if (errno == ENOENT || errno == ENXIO || + errno == ENODEV || errno == EACCES) + PyErr_SetString(PyExc_NotImplementedError, + "/dev/urandom (or equivalent) not found"); + /* otherwise, keep the OSError exception raised by _Py_open() */ + return -1; + } + if (urandom_cache.fd >= 0) { + /* urandom_fd was initialized by another thread while we were + not holding the GIL, keep it. */ + close(fd); + fd = urandom_cache.fd; + } + else { + if (_Py_fstat(fd, &st)) { + close(fd); + return -1; + } + else { + urandom_cache.fd = fd; + urandom_cache.st_dev = st.st_dev; + urandom_cache.st_ino = st.st_ino; + } + } + } + + do { + n = _Py_read(fd, buffer, (size_t)size); + if (n == -1) + return -1; + if (n == 0) { + PyErr_Format(PyExc_RuntimeError, + "Failed to read %zi bytes from /dev/urandom", + size); + return -1; + } + + buffer += n; + size -= n; + } while (0 < size); + + return 0; +} + +static void +dev_urandom_close(void) +{ + if (urandom_cache.fd >= 0) { + close(urandom_cache.fd); + urandom_cache.fd = -1; + } +} + +#endif + +/* Fill buffer with pseudo-random bytes generated by a linear congruent + generator (LCG): + + x(n+1) = (x(n) * 214013 + 2531011) % 2^32 + + Use bits 23..16 of x(n) to generate a byte. */ +static void +lcg_urandom(unsigned int x0, unsigned char *buffer, size_t size) +{ + size_t index; + unsigned int x; + + x = x0; + for (index=0; index < size; index++) { + x *= 214013; + x += 2531011; + /* modulo 2 ^ (8 * sizeof(int)) */ + buffer[index] = (x >> 16) & 0xff; + } +} + +/* Fill buffer with size pseudo-random bytes from the operating system random + number generator (RNG). It is suitable for most cryptographic purposes + except long living private keys for asymmetric encryption. + + Return 0 on success, raise an exception and return -1 on error. */ +int +_PyOS_URandom(void *buffer, Py_ssize_t size) +{ + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "negative argument not allowed"); + return -1; + } + if (size == 0) + return 0; + +#ifdef MS_WINDOWS + return win32_urandom((unsigned char *)buffer, size, 1); +#elif defined(PY_GETENTROPY) + return py_getentropy(buffer, size, 0); +#else + return dev_urandom_python((char*)buffer, size); +#endif +} + +int Py_ReadHashSeed(char *seed_text, + int *use_hash_seed, + unsigned long *hash_seed) +{ + Py_BUILD_ASSERT(sizeof(_Py_HashSecret_t) == sizeof(_Py_HashSecret.uc)); + /* Convert a text seed to a numeric one */ + if (seed_text && *seed_text != '\0' && strcmp(seed_text, "random") != 0) { + char *endptr = seed_text; + unsigned long seed; + seed = strtoul(seed_text, &endptr, 10); + if (*endptr != '\0' + || seed > 4294967295UL + || (errno == ERANGE && seed == ULONG_MAX)) + { + return -1; + } + /* Use a specific hash */ + *use_hash_seed = 1; + *hash_seed = seed; + } + else { + /* Use a random hash */ + *use_hash_seed = 0; + *hash_seed = 0; + } + return 0; +} + +static void +init_hash_secret(int use_hash_seed, + unsigned long hash_seed) +{ + void *secret = &_Py_HashSecret; + Py_ssize_t secret_size = sizeof(_Py_HashSecret_t); + + if (_Py_HashSecret_Initialized) + return; + _Py_HashSecret_Initialized = 1; + + if (use_hash_seed) { + if (hash_seed == 0) { + /* disable the randomized hash */ + memset(secret, 0, secret_size); + } + else { + /* use the specified hash seed */ + lcg_urandom(hash_seed, secret, secret_size); + } + } + else { + /* use a random hash seed */ +#ifdef MS_WINDOWS + (void)win32_urandom(secret, secret_size, 0); +#elif defined(PY_GETENTROPY) + (void)py_getentropy(secret, secret_size, 1); +#else + dev_urandom_noraise(secret, secret_size); +#endif + } +} + +void +_Py_HashRandomization_Init(_PyCoreConfig *core_config) +{ + char *seed_text; + int use_hash_seed = core_config->use_hash_seed; + unsigned long hash_seed = core_config->hash_seed; + + if (use_hash_seed < 0) { + seed_text = Py_GETENV("PYTHONHASHSEED"); + if (Py_ReadHashSeed(seed_text, &use_hash_seed, &hash_seed) < 0) { + Py_FatalError("PYTHONHASHSEED must be \"random\" or an integer " + "in range [0; 4294967295]"); + } + core_config->use_hash_seed = use_hash_seed; + core_config->hash_seed = hash_seed; + } + init_hash_secret(use_hash_seed, hash_seed); +} + +void +_Py_HashRandomization_Fini(void) +{ +#ifdef MS_WINDOWS + if (hCryptProv) { + CryptReleaseContext(hCryptProv, 0); + hCryptProv = 0; + } +#elif defined(PY_GETENTROPY) + /* nothing to clean */ +#else + dev_urandom_close(); +#endif +} diff -r 37975d542715 Python/pylifecycle.c --- a/Python/pylifecycle.c Sat Jun 04 00:53:32 2016 +0300 +++ b/Python/pylifecycle.c Fri Jun 03 16:52:14 2016 -0700 @@ -66,6 +66,7 @@ extern void _PyHash_Fini(void); extern int _PyTraceMalloc_Init(void); extern int _PyTraceMalloc_Fini(void); +extern void _Py_ReadyTypes(void); #ifdef WITH_THREAD extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); @@ -105,14 +106,24 @@ return PyImport_ImportModule("warnings"); } -static int initialized = 0; -/* API to access the initialized flag -- useful for esoteric use */ +/* APIs to access the initialization flags + * + * Can be called prior to Py_Initialize. + */ +int _Py_CoreInitialized = 0; +int _Py_Initialized = 0; + +int +_Py_IsCoreInitialized(void) +{ + return _Py_CoreInitialized; +} int Py_IsInitialized(void) { - return initialized; + return _Py_Initialized; } /* Helper to allow an embedding application to override the normal @@ -232,7 +243,7 @@ } static void -import_init(PyInterpreterState *interp, PyObject *sysmod) +initimport(PyInterpreterState *interp, PyObject *sysmod) { PyObject *importlib; PyObject *impmod; @@ -282,21 +293,96 @@ _PyImportZip_Init(); } +static void +initexternalimport(PyInterpreterState *interp) +{ + PyObject *value; + value = PyObject_CallMethod(interp->importlib, + "_install_external_importers", ""); + if (value == NULL) { + PyErr_Print(); + Py_FatalError("Py_EndInitialization: external importer setup failed"); + } +} -void -_Py_InitializeEx_Private(int install_sigs, int install_importlib) + +/* Global initializations. Can be undone by Py_Finalize(). Don't + call this twice without an intervening Py_Finalize() call. + + Every call to Py_InitializeCore, Py_Initialize or Py_InitializeEx + must have a corresponding call to Py_Finalize. + + Locking: you must hold the interpreter lock while calling these APIs. + (If the lock has not yet been initialized, that's equivalent to + having the lock, but you cannot use multiple threads.) + +*/ + +/* Begin interpreter initialization + * + * On return, the first thread and interpreter state have been created, + * but the compiler, signal handling, multithreading and + * multiple interpreter support, and codec infrastructure are not yet + * available. + * + * The import system will support builtin and frozen modules only. + * The only supported io is writing to sys.stderr + * + * If any operation invoked by this function fails, a fatal error is + * issued and the function does not return. + * + * Any code invoked from this function should *not* assume it has access + * to the Python C API (unless the API is explicitly listed as being + * safe to call without calling Py_Initialize first) + */ + +/* TODO: Progresively move functionality from Py_BeginInitialization to + * Py_ReadConfig and Py_EndInitialization + */ + +/* #define _INIT_DEBUG_PRINT(msg) printf(msg) */ +#define _INIT_DEBUG_PRINT(msg) + +void _Py_InitializeCore(const _PyCoreConfig *config) { PyInterpreterState *interp; PyThreadState *tstate; PyObject *bimod, *sysmod, *pstderr; char *p; - extern void _Py_ReadyTypes(void); + _PyCoreConfig core_config = _PyCoreConfig_INIT; + _PyMainInterpreterConfig preinit_config = _PyMainInterpreterConfig_INIT; - if (initialized) - return; - initialized = 1; + if (config != NULL) { + core_config = *config; + } + + if (_Py_Initialized) { + Py_FatalError("Py_InitializeCore: main interpreter already initialized"); + } + if (_Py_CoreInitialized) { + Py_FatalError("Py_InitializeCore: runtime core already initialized"); + } + + /* Py_Finalize leaves _Py_Finalizing set in order to help daemon + * threads behave a little more gracefully at interpreter shutdown. + * We clobber it here so the new interpreter can start with a clean + * slate. + * + * However, this may still lead to misbehaviour if there are daemon + * threads still hanging around from a previous Py_Initialize/Finalize + * pair :( + */ _Py_Finalizing = NULL; +#ifdef WITH_THREAD + /* We can't call _PyEval_FiniThreads() in Py_Finalize because + destroying the GIL might fail when it is being referenced from + another running thread (see issue #9901). + Instead we destroy the previously created GIL here, which ensures + that we can call Py_Initialize / Py_Finalize multiple times. */ + _PyEval_FiniThreads(); +#endif /* WITH_THREAD */ + #if defined(HAVE_LANGINFO_H) && defined(HAVE_SETLOCALE) /* Set up the LC_CTYPE locale, so we can obtain the locale's charset without having to switch @@ -317,118 +403,217 @@ if ((p = Py_GETENV("PYTHONHASHSEED")) && *p != '\0') Py_HashRandomizationFlag = add_flag(Py_HashRandomizationFlag, p); - _PyRandom_Init(); + _INIT_DEBUG_PRINT("INITIALISING HASH RANDOMISATION\n"); + _Py_HashRandomization_Init(&core_config); + if (!core_config.use_hash_seed || core_config.hash_seed) { + /* Random or non-zero hash seed */ + Py_HashRandomizationFlag = 1; + } + + _INIT_DEBUG_PRINT("MAKING FIRST INTERPRETER\n"); interp = PyInterpreterState_New(); if (interp == NULL) - Py_FatalError("Py_Initialize: can't make first interpreter"); + Py_FatalError("Py_InitializeCore: can't make main interpreter"); + interp->core_config = core_config; + interp->config = preinit_config; + _INIT_DEBUG_PRINT("MAKING FIRST THREAD STATE\n"); tstate = PyThreadState_New(interp); if (tstate == NULL) - Py_FatalError("Py_Initialize: can't make first thread"); + Py_FatalError("Py_InitializeCore: can't make first thread"); (void) PyThreadState_Swap(tstate); + _INIT_DEBUG_PRINT("INITIALISING GIL STATE API\n"); #ifdef WITH_THREAD /* We can't call _PyEval_FiniThreads() in Py_FinalizeEx because - destroying the GIL might fail when it is being referenced from - another running thread (see issue #9901). - Instead we destroy the previously created GIL here, which ensures that we can call Py_Initialize / Py_FinalizeEx multiple times. */ - _PyEval_FiniThreads(); - /* Auto-thread-state API */ _PyGILState_Init(interp, tstate); #endif /* WITH_THREAD */ + _INIT_DEBUG_PRINT("PREPARING BUILTIN TYPES\n"); _Py_ReadyTypes(); if (!_PyFrame_Init()) - Py_FatalError("Py_Initialize: can't init frames"); + Py_FatalError("Py_InitializeCore: can't init frames"); if (!_PyLong_Init()) - Py_FatalError("Py_Initialize: can't init longs"); + Py_FatalError("Py_InitializeCore: can't init longs"); if (!PyByteArray_Init()) - Py_FatalError("Py_Initialize: can't init bytearray"); + Py_FatalError("Py_InitializeCore: can't init bytearray"); if (!_PyFloat_Init()) - Py_FatalError("Py_Initialize: can't init float"); + Py_FatalError("Py_InitializeCore: can't init float"); + _INIT_DEBUG_PRINT("CREATING MODULES CACHE\n"); interp->modules = PyDict_New(); if (interp->modules == NULL) - Py_FatalError("Py_Initialize: can't make modules dictionary"); + Py_FatalError("Py_InitializeCore: can't make modules dictionary"); + _INIT_DEBUG_PRINT("INITIALISING UNICODE SUPPORT\n"); /* Init Unicode implementation; relies on the codec registry */ if (_PyUnicode_Init() < 0) - Py_FatalError("Py_Initialize: can't initialize unicode"); + Py_FatalError("Py_InitializeCore: can't initialize unicode"); + + _INIT_DEBUG_PRINT("INITIALISING STRUCTSEQ\n"); if (_PyStructSequence_Init() < 0) - Py_FatalError("Py_Initialize: can't initialize structseq"); + Py_FatalError("Py_InitializeCore: can't initialize structseq"); + _INIT_DEBUG_PRINT("INITIALISING BUILTINS MODULE\n"); bimod = _PyBuiltin_Init(); if (bimod == NULL) - Py_FatalError("Py_Initialize: can't initialize builtins modules"); + Py_FatalError("Py_InitializeCore: can't initialize builtins modules"); _PyImport_FixupBuiltin(bimod, "builtins"); interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) - Py_FatalError("Py_Initialize: can't initialize builtins dict"); + Py_FatalError("Py_InitializeCore: can't initialize builtins dict"); Py_INCREF(interp->builtins); + _INIT_DEBUG_PRINT("INITIALISING BUILTIN EXCEPTIONS\n"); /* initialize builtin exceptions */ _PyExc_Init(bimod); - sysmod = _PySys_Init(); + _INIT_DEBUG_PRINT("CREATING SYS MODULE\n"); + sysmod = _PySys_BeginInit(); if (sysmod == NULL) - Py_FatalError("Py_Initialize: can't initialize sys"); + Py_FatalError("Py_InitializeCore: can't initialize sys"); + _INIT_DEBUG_PRINT("CACHING REFERENCE TO SYS NAMESPACE\n"); interp->sysdict = PyModule_GetDict(sysmod); if (interp->sysdict == NULL) - Py_FatalError("Py_Initialize: can't initialize sys dict"); + Py_FatalError("Py_InitializeCore: can't initialize sys dict"); Py_INCREF(interp->sysdict); + _INIT_DEBUG_PRINT("FIX UP SYS MODULE IMPORT METADATA\n"); _PyImport_FixupBuiltin(sysmod, "sys"); - PySys_SetPath(Py_GetPath()); + _INIT_DEBUG_PRINT("PUBLISHING REFERENCE TO MODULES CACHE\n"); PyDict_SetItemString(interp->sysdict, "modules", interp->modules); + _INIT_DEBUG_PRINT("INITIALISING PRELIMINARY STDERR\n"); /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); if (pstderr == NULL) - Py_FatalError("Py_Initialize: can't set preliminary stderr"); + Py_FatalError("Py_InitializeCore: can't set preliminary stderr"); _PySys_SetObjectId(&PyId_stderr, pstderr); PySys_SetObject("__stderr__", pstderr); Py_DECREF(pstderr); + _INIT_DEBUG_PRINT("INITIALISING CORE IMPORT SYSTEM\n"); _PyImport_Init(); + _INIT_DEBUG_PRINT("INITIALISING IMPORT HOOK SYSTEM\n"); _PyImportHooks_Init(); + _INIT_DEBUG_PRINT("INITIALISING WARNINGS MODULE\n"); /* Initialize _warnings. */ _PyWarnings_Init(); - if (!install_importlib) - return; + _INIT_DEBUG_PRINT("INITIALISING BUILTIN AND FROZEN IMPORTS\n"); + /* This call sets up builtin and frozen import support */ + if (!interp->core_config._disable_importlib) { + initimport(interp, sysmod); + } + + /* Only when we get here is the runtime core fully initialized */ + _Py_CoreInitialized = 1; +} + +/* Read configuration settings from standard locations + * + * This function doesn't make any changes to the interpreter state - it + * merely populates any missing configuration settings. This allows an + * embedding application to completely override a config option by + * setting it before calling this function, or else modify the default + * setting before passing the fully populated config to Py_EndInitialization. + * + * More advanced selective initialization tricks are possible by calling + * this function multiple times with various preconfigured settings. + */ + +int _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config) +{ + /* Signal handlers are installed by default */ + if (config->install_signal_handlers < 0) { + config->install_signal_handlers = 1; + } + + return 0; +} + +/* Update interpreter state based on supplied configuration settings + * + * After calling this function, most of the restrictions on the interpreter + * are lifted. The only remaining incomplete settings are those related + * to the main module (sys.argv[0], __main__ metadata) + * + * Calling this when the interpreter is not initializing, is already + * initialized or without a valid current thread state is a fatal error. + * Other errors should be reported as normal Python exceptions with a + * non-zero return code. + */ +int _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) +{ + PyInterpreterState *interp; + PyThreadState *tstate; + + if (!_Py_CoreInitialized) { + Py_FatalError("Py_InitializeMainInterpreter: runtime core not initialized"); + } + if (_Py_Initialized) { + Py_FatalError("Py_InitializeMainInterpreter: main interpreter already initialized"); + } + + /* Get current thread state and interpreter pointer */ + tstate = PyThreadState_GET(); + if (!tstate) + Py_FatalError("Py_InitializeMainInterpreter: failed to read thread state"); + interp = tstate->interp; + if (!tstate) + Py_FatalError("Py_InitializeMainInterpreter: failed to get interpreter"); + + /* Now finish configuring the main interpreter */ + interp->config = *config; + + if (interp->core_config._disable_importlib) { + /* Special mode for freeze_importlib: run with no import system + * + * This means anything which needs support from extension modules + * or pure Python code in the standard library won't work. + */ + _Py_Initialized = 1; + return 0; + } + /* TODO: Report exceptions rather than fatal errors below here */ if (_PyTime_Init() < 0) - Py_FatalError("Py_Initialize: can't initialize time"); + Py_FatalError("Py_InitializeMainInterpreter: can't initialize time"); - import_init(interp, sysmod); + /* Finish setting up the sys module and import system */ + if (_PySys_EndInit(interp->sysdict) < 0) + Py_FatalError("Py_InitializeMainInterpreter: can't finish initializing sys"); + /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */ + PySys_SetPath(Py_GetPath()); + initexternalimport(interp); /* initialize the faulthandler module */ if (_PyFaulthandler_Init()) - Py_FatalError("Py_Initialize: can't initialize faulthandler"); + Py_FatalError("Py_InitializeMainInterpreter: can't initialize faulthandler"); if (initfsencoding(interp) < 0) - Py_FatalError("Py_Initialize: unable to load the file system codec"); + Py_FatalError("Py_InitializeMainInterpreter: unable to load the file system codec"); - if (install_sigs) + if (config->install_signal_handlers) initsigs(); /* Signal handling stuff, including initintr() */ if (_PyTraceMalloc_Init() < 0) - Py_FatalError("Py_Initialize: can't initialize tracemalloc"); + Py_FatalError("Py_InitializeMainInterpreter: can't initialize tracemalloc"); initmain(interp); /* Module __main__ */ if (initstdio() < 0) Py_FatalError( - "Py_Initialize: can't initialize sys standard streams"); + "Py_InitializeMainInterpreter: can't initialize sys standard streams"); /* Initialize warnings. */ if (PySys_HasWarnOptions()) { @@ -440,8 +625,32 @@ Py_XDECREF(warnings_module); } + _Py_Initialized = 1; + if (!Py_NoSiteFlag) initsite(); /* Module site */ + + return 0; +} + +#undef _INIT_DEBUG_PRINT + +void +_Py_InitializeEx_Private(int install_sigs, int install_importlib) +{ + _PyCoreConfig core_config = _PyCoreConfig_INIT; + _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; + + /* TODO: Moar config options! */ + core_config.ignore_environment = Py_IgnoreEnvironmentFlag; + core_config._disable_importlib = !install_importlib; + config.install_signal_handlers = install_sigs; + _Py_InitializeCore(&core_config); + /* TODO: Print any exceptions raised by these operations */ + if (_Py_ReadMainInterpreterConfig(&config)) + Py_FatalError("Py_Initialize: Py_ReadMainInterpreterConfig failed"); + if (_Py_InitializeMainInterpreter(&config)) + Py_FatalError("Py_Initialize: Py_InitializeMainInterpreter failed"); } void @@ -531,7 +740,7 @@ PyThreadState *tstate; int status = 0; - if (!initialized) + if (!_Py_Initialized) return status; wait_for_thread_shutdown(); @@ -554,7 +763,8 @@ /* Remaining threads (e.g. daemon threads) will automatically exit after taking the GIL (in PyEval_RestoreThread()). */ _Py_Finalizing = tstate; - initialized = 0; + _Py_Initialized = 0; + _Py_CoreInitialized = 0; /* Flush sys.stdout and sys.stderr */ if (flush_std_files() < 0) { @@ -668,7 +878,7 @@ PyDict_Fini(); PySlice_Fini(); _PyGC_Fini(); - _PyRandom_Fini(); + _Py_HashRandomization_Fini(); /* Cleanup Unicode implementation */ _PyUnicode_Fini(); @@ -743,7 +953,7 @@ PyThreadState *tstate, *save_tstate; PyObject *bimod, *sysmod; - if (!initialized) + if (!_Py_Initialized) Py_FatalError("Py_NewInterpreter: call Py_Initialize first"); /* Issue #10915, #15751: The GIL API doesn't work with multiple @@ -762,6 +972,17 @@ save_tstate = PyThreadState_Swap(tstate); + /* Copy the current interpreter config into the new interpreter */ + if (save_tstate != NULL) { + interp->core_config = save_tstate->interp->core_config; + interp->config = save_tstate->interp->config; + } else { + /* No current thread state, copy from the main interpreter */ + PyInterpreterState *main_interp = PyInterpreterState_Main(); + interp->core_config = main_interp->core_config; + interp->config = main_interp->config; + } + /* XXX The following is lax in error checking */ interp->modules = PyDict_New(); @@ -785,6 +1006,7 @@ if (interp->sysdict == NULL) goto handle_error; Py_INCREF(interp->sysdict); + _PySys_EndInit(interp->sysdict); PySys_SetPath(Py_GetPath()); PyDict_SetItemString(interp->sysdict, "modules", interp->modules); @@ -792,21 +1014,22 @@ infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); if (pstderr == NULL) - Py_FatalError("Py_Initialize: can't set preliminary stderr"); + Py_FatalError("Py_NewInterpreter: can't set preliminary stderr"); _PySys_SetObjectId(&PyId_stderr, pstderr); PySys_SetObject("__stderr__", pstderr); Py_DECREF(pstderr); _PyImportHooks_Init(); - import_init(interp, sysmod); + initimport(interp, sysmod); + initexternalimport(interp); if (initfsencoding(interp) < 0) goto handle_error; if (initstdio() < 0) Py_FatalError( - "Py_Initialize: can't initialize sys standard streams"); + "Py_NewInterpreter: can't initialize sys standard streams"); initmain(interp); if (!Py_NoSiteFlag) initsite(); diff -r 37975d542715 Python/pystate.c --- a/Python/pystate.c Sat Jun 04 00:53:32 2016 +0300 +++ b/Python/pystate.c Fri Jun 03 16:52:14 2016 -0700 @@ -46,6 +46,7 @@ /* The single PyInterpreterState used by this process' GILState implementation */ +/* TODO: Given interp_main, it may be possible to kill this ref */ static PyInterpreterState *autoInterpreterState = NULL; static int autoTLSkey = -1; #else @@ -55,6 +56,7 @@ #endif static PyInterpreterState *interp_head = NULL; +static PyInterpreterState *interp_main = NULL; /* Assuming the current thread holds the GIL, this is the PyThreadState for the current thread. */ @@ -103,6 +105,9 @@ HEAD_LOCK(); interp->next = interp_head; + if (interp_main == NULL) { + interp_main = interp; + } interp_head = interp; HEAD_UNLOCK(); } @@ -159,6 +164,11 @@ if (interp->tstate_head != NULL) Py_FatalError("PyInterpreterState_Delete: remaining threads"); *p = interp->next; + if (interp_main == interp) { + interp_main = NULL; + if (interp_head != NULL) + Py_FatalError("PyInterpreterState_Delete: remaining subinterpreters"); + } HEAD_UNLOCK(); PyMem_RawFree(interp); #ifdef WITH_THREAD @@ -616,6 +626,12 @@ } PyInterpreterState * +PyInterpreterState_Main(void) +{ + return interp_main; +} + +PyInterpreterState * PyInterpreterState_Next(PyInterpreterState *interp) { return interp->next; } diff -r 37975d542715 Python/pythonrun.c --- a/Python/pythonrun.c Sat Jun 04 00:53:32 2016 +0300 +++ b/Python/pythonrun.c Fri Jun 03 16:52:14 2016 -0700 @@ -1,5 +1,12 @@ -/* Python interpreter top-level routines, including init/exit */ +/* Top level execution of Python code (including in __main__) */ + +/* To help control the interfaces between the startup, execution and + * shutdown code, the phases are split across separate modules (boostrap, + * pythonrun, shutdown) + */ + +/* TODO: Cull includes following phase split */ #include "Python.h" @@ -32,6 +39,7 @@ #ifdef MS_WINDOWS #undef BYTE #include "windows.h" +/* TODO: This is duped in shutdown.c Find a way to merge them */ #endif _Py_IDENTIFIER(builtins); @@ -63,7 +71,6 @@ static void err_free(perrdetail *); /* Parse input from a file and execute it */ - int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) @@ -1387,44 +1394,6 @@ } } - -#if defined(USE_STACKCHECK) -#if defined(WIN32) && defined(_MSC_VER) - -/* Stack checking for Microsoft C */ - -#include -#include - -/* - * Return non-zero when we run out of memory on the stack; zero otherwise. - */ -int -PyOS_CheckStack(void) -{ - __try { - /* alloca throws a stack overflow exception if there's - not enough space left on the stack */ - alloca(PYOS_STACK_MARGIN * sizeof(void*)); - return 0; - } __except (GetExceptionCode() == STATUS_STACK_OVERFLOW ? - EXCEPTION_EXECUTE_HANDLER : - EXCEPTION_CONTINUE_SEARCH) { - int errcode = _resetstkoflw(); - if (errcode == 0) - { - Py_FatalError("Could not reset the stack!"); - } - } - return 1; -} - -#endif /* WIN32 && _MSC_VER */ - -/* Alternate implementations can be added here... */ - -#endif /* USE_STACKCHECK */ - /* Deprecated C API functions still provided for binary compatibility */ #undef PyParser_SimpleParseFile diff -r 37975d542715 Python/random.c --- a/Python/random.c Sat Jun 04 00:53:32 2016 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,444 +0,0 @@ -#include "Python.h" -#ifdef MS_WINDOWS -# include -#else -# include -# ifdef HAVE_SYS_STAT_H -# include -# endif -# ifdef HAVE_GETRANDOM -# include -# elif defined(HAVE_GETRANDOM_SYSCALL) -# include -# endif -#endif - -#ifdef Py_DEBUG -int _Py_HashSecret_Initialized = 0; -#else -static int _Py_HashSecret_Initialized = 0; -#endif - -#ifdef MS_WINDOWS -static HCRYPTPROV hCryptProv = 0; - -static int -win32_urandom_init(int raise) -{ - /* Acquire context */ - if (!CryptAcquireContext(&hCryptProv, NULL, NULL, - PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) - goto error; - - return 0; - -error: - if (raise) - PyErr_SetFromWindowsErr(0); - else - Py_FatalError("Failed to initialize Windows random API (CryptoGen)"); - return -1; -} - -/* Fill buffer with size pseudo-random bytes generated by the Windows CryptoGen - API. Return 0 on success, or raise an exception and return -1 on error. */ -static int -win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise) -{ - Py_ssize_t chunk; - - if (hCryptProv == 0) - { - if (win32_urandom_init(raise) == -1) - return -1; - } - - while (size > 0) - { - chunk = size > INT_MAX ? INT_MAX : size; - if (!CryptGenRandom(hCryptProv, (DWORD)chunk, buffer)) - { - /* CryptGenRandom() failed */ - if (raise) - PyErr_SetFromWindowsErr(0); - else - Py_FatalError("Failed to initialized the randomized hash " - "secret using CryptoGen)"); - return -1; - } - buffer += chunk; - size -= chunk; - } - return 0; -} - -#elif defined(HAVE_GETENTROPY) && !defined(sun) -#define PY_GETENTROPY 1 - -/* Fill buffer with size pseudo-random bytes generated by getentropy(). - Return 0 on success, or raise an exception and return -1 on error. - - If fatal is nonzero, call Py_FatalError() instead of raising an exception - on error. */ -static int -py_getentropy(unsigned char *buffer, Py_ssize_t size, int fatal) -{ - while (size > 0) { - Py_ssize_t len = Py_MIN(size, 256); - int res; - - if (!fatal) { - Py_BEGIN_ALLOW_THREADS - res = getentropy(buffer, len); - Py_END_ALLOW_THREADS - - if (res < 0) { - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - } - else { - res = getentropy(buffer, len); - if (res < 0) - Py_FatalError("getentropy() failed"); - } - - buffer += len; - size -= len; - } - return 0; -} - -#else - -/* Issue #25003: Don' use getentropy() on Solaris (available since - * Solaris 11.3), it is blocking whereas os.urandom() should not block. */ -#if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL) -#define PY_GETRANDOM 1 - -static int -py_getrandom(void *buffer, Py_ssize_t size, int raise) -{ - /* Is getrandom() supported by the running kernel? - * Need Linux kernel 3.17 or newer, or Solaris 11.3 or newer */ - static int getrandom_works = 1; - /* Use non-blocking /dev/urandom device. On Linux at boot, the getrandom() - * syscall blocks until /dev/urandom is initialized with enough entropy. */ - const int flags = 0; - int n; - - if (!getrandom_works) - return 0; - - while (0 < size) { -#ifdef sun - /* Issue #26735: On Solaris, getrandom() is limited to returning up - to 1024 bytes */ - n = Py_MIN(size, 1024); -#else - n = size; -#endif - - errno = 0; -#ifdef HAVE_GETRANDOM - if (raise) { - Py_BEGIN_ALLOW_THREADS - n = getrandom(buffer, n, flags); - Py_END_ALLOW_THREADS - } - else { - n = getrandom(buffer, n, flags); - } -#else - /* On Linux, use the syscall() function because the GNU libc doesn't - * expose the Linux getrandom() syscall yet. See: - * https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */ - if (raise) { - Py_BEGIN_ALLOW_THREADS - n = syscall(SYS_getrandom, buffer, n, flags); - Py_END_ALLOW_THREADS - } - else { - n = syscall(SYS_getrandom, buffer, n, flags); - } -#endif - - if (n < 0) { - if (errno == ENOSYS) { - getrandom_works = 0; - return 0; - } - - if (errno == EINTR) { - if (PyErr_CheckSignals()) { - if (!raise) - Py_FatalError("getrandom() interrupted by a signal"); - return -1; - } - /* retry getrandom() */ - continue; - } - - if (raise) - PyErr_SetFromErrno(PyExc_OSError); - else - Py_FatalError("getrandom() failed"); - return -1; - } - - buffer += n; - size -= n; - } - return 1; -} -#endif - -static struct { - int fd; - dev_t st_dev; - ino_t st_ino; -} urandom_cache = { -1 }; - - -/* Read size bytes from /dev/urandom into buffer. - Call Py_FatalError() on error. */ -static void -dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size) -{ - int fd; - Py_ssize_t n; - - assert (0 < size); - -#ifdef PY_GETRANDOM - if (py_getrandom(buffer, size, 0) == 1) - return; - /* getrandom() is not supported by the running kernel, fall back - * on reading /dev/urandom */ -#endif - - fd = _Py_open_noraise("/dev/urandom", O_RDONLY); - if (fd < 0) - Py_FatalError("Failed to open /dev/urandom"); - - while (0 < size) - { - do { - n = read(fd, buffer, (size_t)size); - } while (n < 0 && errno == EINTR); - if (n <= 0) - { - /* stop on error or if read(size) returned 0 */ - Py_FatalError("Failed to read bytes from /dev/urandom"); - break; - } - buffer += n; - size -= (Py_ssize_t)n; - } - close(fd); -} - -/* Read size bytes from /dev/urandom into buffer. - Return 0 on success, raise an exception and return -1 on error. */ -static int -dev_urandom_python(char *buffer, Py_ssize_t size) -{ - int fd; - Py_ssize_t n; - struct _Py_stat_struct st; -#ifdef PY_GETRANDOM - int res; -#endif - - if (size <= 0) - return 0; - -#ifdef PY_GETRANDOM - res = py_getrandom(buffer, size, 1); - if (res < 0) - return -1; - if (res == 1) - return 0; - /* getrandom() is not supported by the running kernel, fall back - * on reading /dev/urandom */ -#endif - - if (urandom_cache.fd >= 0) { - /* Does the fd point to the same thing as before? (issue #21207) */ - if (_Py_fstat_noraise(urandom_cache.fd, &st) - || st.st_dev != urandom_cache.st_dev - || st.st_ino != urandom_cache.st_ino) { - /* Something changed: forget the cached fd (but don't close it, - since it probably points to something important for some - third-party code). */ - urandom_cache.fd = -1; - } - } - if (urandom_cache.fd >= 0) - fd = urandom_cache.fd; - else { - fd = _Py_open("/dev/urandom", O_RDONLY); - if (fd < 0) { - if (errno == ENOENT || errno == ENXIO || - errno == ENODEV || errno == EACCES) - PyErr_SetString(PyExc_NotImplementedError, - "/dev/urandom (or equivalent) not found"); - /* otherwise, keep the OSError exception raised by _Py_open() */ - return -1; - } - if (urandom_cache.fd >= 0) { - /* urandom_fd was initialized by another thread while we were - not holding the GIL, keep it. */ - close(fd); - fd = urandom_cache.fd; - } - else { - if (_Py_fstat(fd, &st)) { - close(fd); - return -1; - } - else { - urandom_cache.fd = fd; - urandom_cache.st_dev = st.st_dev; - urandom_cache.st_ino = st.st_ino; - } - } - } - - do { - n = _Py_read(fd, buffer, (size_t)size); - if (n == -1) - return -1; - if (n == 0) { - PyErr_Format(PyExc_RuntimeError, - "Failed to read %zi bytes from /dev/urandom", - size); - return -1; - } - - buffer += n; - size -= n; - } while (0 < size); - - return 0; -} - -static void -dev_urandom_close(void) -{ - if (urandom_cache.fd >= 0) { - close(urandom_cache.fd); - urandom_cache.fd = -1; - } -} - -#endif - -/* Fill buffer with pseudo-random bytes generated by a linear congruent - generator (LCG): - - x(n+1) = (x(n) * 214013 + 2531011) % 2^32 - - Use bits 23..16 of x(n) to generate a byte. */ -static void -lcg_urandom(unsigned int x0, unsigned char *buffer, size_t size) -{ - size_t index; - unsigned int x; - - x = x0; - for (index=0; index < size; index++) { - x *= 214013; - x += 2531011; - /* modulo 2 ^ (8 * sizeof(int)) */ - buffer[index] = (x >> 16) & 0xff; - } -} - -/* Fill buffer with size pseudo-random bytes from the operating system random - number generator (RNG). It is suitable for most cryptographic purposes - except long living private keys for asymmetric encryption. - - Return 0 on success, raise an exception and return -1 on error. */ -int -_PyOS_URandom(void *buffer, Py_ssize_t size) -{ - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "negative argument not allowed"); - return -1; - } - if (size == 0) - return 0; - -#ifdef MS_WINDOWS - return win32_urandom((unsigned char *)buffer, size, 1); -#elif defined(PY_GETENTROPY) - return py_getentropy(buffer, size, 0); -#else - return dev_urandom_python((char*)buffer, size); -#endif -} - -void -_PyRandom_Init(void) -{ - char *env; - unsigned char *secret = (unsigned char *)&_Py_HashSecret.uc; - Py_ssize_t secret_size = sizeof(_Py_HashSecret_t); - Py_BUILD_ASSERT(sizeof(_Py_HashSecret_t) == sizeof(_Py_HashSecret.uc)); - - if (_Py_HashSecret_Initialized) - return; - _Py_HashSecret_Initialized = 1; - - /* - Hash randomization is enabled. Generate a per-process secret, - using PYTHONHASHSEED if provided. - */ - - env = Py_GETENV("PYTHONHASHSEED"); - if (env && *env != '\0' && strcmp(env, "random") != 0) { - char *endptr = env; - unsigned long seed; - seed = strtoul(env, &endptr, 10); - if (*endptr != '\0' - || seed > 4294967295UL - || (errno == ERANGE && seed == ULONG_MAX)) - { - Py_FatalError("PYTHONHASHSEED must be \"random\" or an integer " - "in range [0; 4294967295]"); - } - if (seed == 0) { - /* disable the randomized hash */ - memset(secret, 0, secret_size); - } - else { - lcg_urandom(seed, secret, secret_size); - } - } - else { -#ifdef MS_WINDOWS - (void)win32_urandom(secret, secret_size, 0); -#elif defined(PY_GETENTROPY) - (void)py_getentropy(secret, secret_size, 1); -#else - dev_urandom_noraise(secret, secret_size); -#endif - } -} - -void -_PyRandom_Fini(void) -{ -#ifdef MS_WINDOWS - if (hCryptProv) { - CryptReleaseContext(hCryptProv, 0); - hCryptProv = 0; - } -#elif defined(PY_GETENTROPY) - /* nothing to clean */ -#else - dev_urandom_close(); -#endif -} diff -r 37975d542715 Python/sysmodule.c --- a/Python/sysmodule.c Sat Jun 04 00:53:32 2016 +0300 +++ b/Python/sysmodule.c Fri Jun 03 16:52:14 2016 -0700 @@ -1716,16 +1716,7 @@ NULL }; -PyObject * -_PySys_Init(void) -{ - PyObject *m, *sysdict, *version_info; - int res; - - m = PyModule_Create(&sysmodule); - if (m == NULL) - return NULL; - sysdict = PyModule_GetDict(m); +/* Updating the sys namespace, returning NULL pointer on error */ #define SET_SYS_FROM_STRING_BORROW(key, value) \ do { \ PyObject *v = (value); \ @@ -1748,6 +1739,17 @@ } \ } while (0) +PyObject * +_PySys_BeginInit(void) +{ + PyObject *m, *sysdict, *version_info; + int res; + + m = PyModule_Create(&sysmodule); + if (m == NULL) + return NULL; + sysdict = PyModule_GetDict(m); + /* Check that stdin is not a directory Using shell redirection, you can redirect stdin to a directory, crashing the Python interpreter. Catch this common mistake here @@ -1779,25 +1781,12 @@ SET_SYS_FROM_STRING("_mercurial", Py_BuildValue("(szz)", "CPython", _Py_hgidentifier(), _Py_hgversion())); - SET_SYS_FROM_STRING("dont_write_bytecode", - PyBool_FromLong(Py_DontWriteBytecodeFlag)); SET_SYS_FROM_STRING("api_version", PyLong_FromLong(PYTHON_API_VERSION)); SET_SYS_FROM_STRING("copyright", PyUnicode_FromString(Py_GetCopyright())); SET_SYS_FROM_STRING("platform", PyUnicode_FromString(Py_GetPlatform())); - SET_SYS_FROM_STRING("executable", - PyUnicode_FromWideChar( - Py_GetProgramFullPath(), -1)); - SET_SYS_FROM_STRING("prefix", - PyUnicode_FromWideChar(Py_GetPrefix(), -1)); - SET_SYS_FROM_STRING("exec_prefix", - PyUnicode_FromWideChar(Py_GetExecPrefix(), -1)); - SET_SYS_FROM_STRING("base_prefix", - PyUnicode_FromWideChar(Py_GetPrefix(), -1)); - SET_SYS_FROM_STRING("base_exec_prefix", - PyUnicode_FromWideChar(Py_GetExecPrefix(), -1)); SET_SYS_FROM_STRING("maxsize", PyLong_FromSsize_t(PY_SSIZE_T_MAX)); SET_SYS_FROM_STRING("float_info", @@ -1833,17 +1822,6 @@ SET_SYS_FROM_STRING("abiflags", PyUnicode_FromString(ABIFLAGS)); #endif - if (warnoptions == NULL) { - warnoptions = PyList_New(0); - if (warnoptions == NULL) - return NULL; - } - else { - Py_INCREF(warnoptions); - } - SET_SYS_FROM_STRING_BORROW("warnoptions", warnoptions); - - SET_SYS_FROM_STRING_BORROW("_xoptions", get_xoptions()); /* version_info */ if (VersionInfoType.tp_name == NULL) { @@ -1868,13 +1846,8 @@ if (PyStructSequence_InitType2(&FlagsType, &flags_desc) < 0) return NULL; } + /* Set flags to their default values */ SET_SYS_FROM_STRING("flags", make_flags()); - /* prevent user from creating new instances */ - FlagsType.tp_init = NULL; - FlagsType.tp_new = NULL; - res = PyDict_DelItemString(FlagsType.tp_dict, "__new__"); - if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) - PyErr_Clear(); #if defined(MS_WINDOWS) /* getwindowsversion */ @@ -1903,13 +1876,89 @@ SET_SYS_FROM_STRING("thread_info", PyThread_GetInfo()); #endif -#undef SET_SYS_FROM_STRING -#undef SET_SYS_FROM_STRING_BORROW if (PyErr_Occurred()) return NULL; return m; } +#undef SET_SYS_FROM_STRING +#undef SET_SYS_FROM_STRING_BORROW + +/* Updating the sys namespace, returning integer error codes */ +#define SET_SYS_FROM_STRING_BORROW_INT_RESULT(key, value) \ + do { \ + PyObject *v = (value); \ + if (v == NULL) \ + return -1; \ + res = PyDict_SetItemString(sysdict, key, v); \ + if (res < 0) { \ + return res; \ + } \ + } while (0) +#define SET_SYS_FROM_STRING_INT_RESULT(key, value) \ + do { \ + PyObject *v = (value); \ + if (v == NULL) \ + return -1; \ + res = PyDict_SetItemString(sysdict, key, v); \ + Py_DECREF(v); \ + if (res < 0) { \ + return res; \ + } \ + } while (0) + +int +_PySys_EndInit(PyObject *sysdict) +{ + int res; + + /* Set flags to their final values */ + SET_SYS_FROM_STRING_INT_RESULT("flags", make_flags()); + /* prevent user from creating new instances */ + FlagsType.tp_init = NULL; + FlagsType.tp_new = NULL; + res = PyDict_DelItemString(FlagsType.tp_dict, "__new__"); + if (res < 0) { + if (!PyErr_ExceptionMatches(PyExc_KeyError)) { + return res; + } + PyErr_Clear(); + } + + SET_SYS_FROM_STRING_INT_RESULT("dont_write_bytecode", + PyBool_FromLong(Py_DontWriteBytecodeFlag)); + SET_SYS_FROM_STRING_INT_RESULT("executable", + PyUnicode_FromWideChar( + Py_GetProgramFullPath(), -1)); + SET_SYS_FROM_STRING_INT_RESULT("prefix", + PyUnicode_FromWideChar(Py_GetPrefix(), -1)); + SET_SYS_FROM_STRING_INT_RESULT("exec_prefix", + PyUnicode_FromWideChar(Py_GetExecPrefix(), -1)); + SET_SYS_FROM_STRING_INT_RESULT("base_prefix", + PyUnicode_FromWideChar(Py_GetPrefix(), -1)); + SET_SYS_FROM_STRING_INT_RESULT("base_exec_prefix", + PyUnicode_FromWideChar(Py_GetExecPrefix(), -1)); + + if (warnoptions == NULL) { + warnoptions = PyList_New(0); + if (warnoptions == NULL) + return -1; + } + else { + Py_INCREF(warnoptions); + } + SET_SYS_FROM_STRING_BORROW_INT_RESULT("warnoptions", warnoptions); + + SET_SYS_FROM_STRING_BORROW_INT_RESULT("_xoptions", get_xoptions()); + + if (PyErr_Occurred()) + return -1; + return 0; +} + +#undef SET_SYS_FROM_STRING_INT_RESULT +#undef SET_SYS_FROM_STRING_BORROW_INT_RESULT + static PyObject * makepathobject(const wchar_t *path, wchar_t delim) {