comparing with /home/esnow/projects/.external/.cpython searching for changes changeset: 76512:5296d2739a31 tag: issue13959_magic.diff tag: qbase tag: qtip tag: tip user: Eric Snow date: Tue Apr 24 05:34:12 2012 +0000 summary: [issue 13959] port MAGIC to importlib diff --git a/Lib/imp.py b/Lib/imp.py --- a/Lib/imp.py +++ b/Lib/imp.py @@ -11,7 +11,7 @@ init_builtin, init_frozen, is_builtin, is_frozen, _fix_co_filename) # Could move out of _imp, but not worth the code -from _imp import get_magic, get_tag +from _imp import get_tag # Can (probably) move to importlib from _imp import get_suffixes # Should be re-implemented here (and mostly deprecated) @@ -194,3 +194,8 @@ encoding = tokenize.detect_encoding(file.readline)[0] file = open(file_path, mode, encoding=encoding) return file, file_path, (suffix, mode, type_) + + +def get_magic(): + """Return the magic number for .pyc or .pyo files.""" + return _bootstrap._MAGIC_BYTES diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -175,6 +175,96 @@ code_type = type(_wrap.__code__) +# legacy imp interface support ################################################ + +# Magic word to reject .pyc files generated by other Python versions. +# It should change for each incompatible change to the bytecode. +# +# The value of CR and LF is incorporated so if you ever read or write +# a .pyc file in text mode the magic number will be wrong; also, the +# Apple MPW compiler swaps their values, botching string constants. +# +# There were a variety of old schemes for setting the magic number. +# The current working scheme is to increment the previous value by +# 10. +# +# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic +# number also includes a new "magic tag", i.e. a human readable string used +# to represent the magic number in __pycache__ directories. When you change +# the magic number, you must also set a new unique magic tag. Generally this +# can be named after the Python major version of the magic number bump, but +# it can really be anything, as long as it's different than anything else +# that's come before. The tags are included in the following table, starting +# with Python 3.2a0. +# +# Known values: +# Python 1.5: 20121 +# Python 1.5.1: 20121 +# Python 1.5.2: 20121 +# Python 1.6: 50428 +# Python 2.0: 50823 +# Python 2.0.1: 50823 +# Python 2.1: 60202 +# Python 2.1.1: 60202 +# Python 2.1.2: 60202 +# Python 2.2: 60717 +# Python 2.3a0: 62011 +# Python 2.3a0: 62021 +# Python 2.3a0: 62011 (!) +# Python 2.4a0: 62041 +# Python 2.4a3: 62051 +# Python 2.4b1: 62061 +# Python 2.5a0: 62071 +# Python 2.5a0: 62081 (ast-branch) +# Python 2.5a0: 62091 (with) +# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) +# Python 2.5b3: 62101 (fix wrong code: for x, in ...) +# Python 2.5b3: 62111 (fix wrong code: x += yield) +# Python 2.5c1: 62121 (fix wrong lnotab with for loops and +# storing constants that should have been removed) +# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) +# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) +# Python 2.6a1: 62161 (WITH_CLEANUP optimization) +# Python 3000: 3000 +# 3010 (removed UNARY_CONVERT) +# 3020 (added BUILD_SET) +# 3030 (added keyword-only parameters) +# 3040 (added signature annotations) +# 3050 (print becomes a function) +# 3060 (PEP 3115 metaclass syntax) +# 3061 (string literals become unicode) +# 3071 (PEP 3109 raise changes) +# 3081 (PEP 3137 make __file__ and __name__ unicode) +# 3091 (kill str8 interning) +# 3101 (merge from 2.6a0, see 62151) +# 3103 (__file__ points to source file) +# Python 3.0a4: 3111 (WITH_CLEANUP optimization). +# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) +# Python 3.1a0: 3141 (optimize list, set and dict comprehensions: +# change LIST_APPEND and SET_ADD, add MAP_ADD) +# Python 3.1a0: 3151 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) +# Python 3.2a0: 3160 (add SETUP_WITH) +# tag: cpython-32 +# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) +# tag: cpython-32 +# Python 3.2a2 3180 (add DELETE_DEREF) +# Python 3.3a0 3190 __class__ super closure changed +# Python 3.3a0 3200 (__qualname__ added) +# 3210 (added size modulo 2**32 to the pyc header) +# Python 3.3a1 3220 (changed PEP 380 implementation) + +# _MAGIC_NUMBER must change whenever the bytecode emitted by the compiler may +# no longer be understood by older implementations of the eval loop (usually +# due to the addition of new opcodes). +_RAW_MAGIC_NUMBER = 3220 + +_MAGIC_BYTES = bytes([_RAW_MAGIC_NUMBER & 0xff, + _RAW_MAGIC_NUMBER >> 8, + ord('\r'), + ord('\n')]) + + def _new_module(name): """Create a new module. @@ -444,7 +534,7 @@ magic = data[:4] raw_timestamp = data[4:8] raw_size = data[8:12] - if magic != _MAGIC_NUMBER: + if magic != _MAGIC_BYTES: msg = 'bad magic number in {!r}: {!r}'.format(fullname, magic) raise ImportError(msg, name=fullname, path=bytecode_path) elif len(raw_timestamp) != 4: @@ -589,7 +679,7 @@ verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): - data = bytearray(_MAGIC_NUMBER) + data = bytearray(_MAGIC_BYTES) data.extend(_w_long(source_mtime)) data.extend(_w_long(len(source_bytes))) data.extend(marshal.dumps(code_object)) @@ -1139,7 +1229,6 @@ return _handle_fromlist(module, fromlist, _gcd_import) -_MAGIC_NUMBER = None # Set in _setup() _TAG = None # Set in _setup() @@ -1191,7 +1280,6 @@ setattr(self_module, 'path_separators', set(path_separators)) # Constants setattr(self_module, '_relax_case', _make_relax_case()) - setattr(self_module, '_MAGIC_NUMBER', _imp_module.get_magic()) setattr(self_module, '_TAG', _imp.get_tag()) if builtin_os == 'nt': SOURCE_SUFFIXES.append('.pyw') diff --git a/Python/import.c b/Python/import.c --- a/Python/import.c +++ b/Python/import.c @@ -26,103 +26,16 @@ #endif -/* Magic word to reject .pyc files generated by other Python versions. - It should change for each incompatible change to the bytecode. - - The value of CR and LF is incorporated so if you ever read or write - a .pyc file in text mode the magic number will be wrong; also, the - Apple MPW compiler swaps their values, botching string constants. - - The magic numbers must be spaced apart at least 2 values, as the - -U interpeter flag will cause MAGIC+1 being used. They have been - odd numbers for some time now. - - There were a variety of old schemes for setting the magic number. - The current working scheme is to increment the previous value by - 10. - - Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic - number also includes a new "magic tag", i.e. a human readable string used - to represent the magic number in __pycache__ directories. When you change - the magic number, you must also set a new unique magic tag. Generally this - can be named after the Python major version of the magic number bump, but - it can really be anything, as long as it's different than anything else - that's come before. The tags are included in the following table, starting - with Python 3.2a0. - - Known values: - Python 1.5: 20121 - Python 1.5.1: 20121 - Python 1.5.2: 20121 - Python 1.6: 50428 - Python 2.0: 50823 - Python 2.0.1: 50823 - Python 2.1: 60202 - Python 2.1.1: 60202 - Python 2.1.2: 60202 - Python 2.2: 60717 - Python 2.3a0: 62011 - Python 2.3a0: 62021 - Python 2.3a0: 62011 (!) - Python 2.4a0: 62041 - Python 2.4a3: 62051 - Python 2.4b1: 62061 - Python 2.5a0: 62071 - Python 2.5a0: 62081 (ast-branch) - Python 2.5a0: 62091 (with) - Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) - Python 2.5b3: 62101 (fix wrong code: for x, in ...) - Python 2.5b3: 62111 (fix wrong code: x += yield) - Python 2.5c1: 62121 (fix wrong lnotab with for loops and - storing constants that should have been removed) - Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) - Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) - Python 2.6a1: 62161 (WITH_CLEANUP optimization) - Python 3000: 3000 - 3010 (removed UNARY_CONVERT) - 3020 (added BUILD_SET) - 3030 (added keyword-only parameters) - 3040 (added signature annotations) - 3050 (print becomes a function) - 3060 (PEP 3115 metaclass syntax) - 3061 (string literals become unicode) - 3071 (PEP 3109 raise changes) - 3081 (PEP 3137 make __file__ and __name__ unicode) - 3091 (kill str8 interning) - 3101 (merge from 2.6a0, see 62151) - 3103 (__file__ points to source file) - Python 3.0a4: 3111 (WITH_CLEANUP optimization). - Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) - Python 3.1a0: 3141 (optimize list, set and dict comprehensions: - change LIST_APPEND and SET_ADD, add MAP_ADD) - Python 3.1a0: 3151 (optimize conditional branches: - introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) - Python 3.2a0: 3160 (add SETUP_WITH) - tag: cpython-32 - Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) - tag: cpython-32 - Python 3.2a2 3180 (add DELETE_DEREF) - Python 3.3a0 3190 __class__ super closure changed - Python 3.3a0 3200 (__qualname__ added) - 3210 (added size modulo 2**32 to the pyc header) - Python 3.3a1 3220 (changed PEP 380 implementation) -*/ - -/* MAGIC must change whenever the bytecode emitted by the compiler may no - longer be understood by older implementations of the eval loop (usually - due to the addition of new opcodes) - TAG must change for each major Python release. The magic number will take +/* TAG must change for each major Python release. The magic number will take care of any bytecode changes that occur during development. */ #define QUOTE(arg) #arg #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3220 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" -/* Current magic word and string tag as globals. */ -static long pyc_magic = MAGIC; +/* Current string tag as global. */ static const char *pyc_tag = TAG; #undef QUOTE #undef STRIFY @@ -588,7 +501,12 @@ long PyImport_GetMagicNumber(void) { - return pyc_magic; + PyInterpreterState *interp = PyThreadState_Get()->interp; + PyObject *pyc_magic = PyObject_GetAttrString(interp->importlib, + "_RAW_MAGIC_NUMBER"); + if (pyc_magic == NULL) + return NULL; + return PyLong_AsLong(pyc_magic); } @@ -1970,25 +1888,6 @@ */ static PyObject * -imp_make_magic(long magic) -{ - char buf[4]; - - buf[0] = (char) ((magic >> 0) & 0xff); - buf[1] = (char) ((magic >> 8) & 0xff); - buf[2] = (char) ((magic >> 16) & 0xff); - buf[3] = (char) ((magic >> 24) & 0xff); - - return PyBytes_FromStringAndSize(buf, 4); -} - -static PyObject * -imp_get_magic(PyObject *self, PyObject *noargs) -{ - return imp_make_magic(pyc_magic); -} - -static PyObject * imp_get_tag(PyObject *self, PyObject *noargs) { return PyUnicode_FromString(pyc_tag); @@ -2188,10 +2087,6 @@ PyDoc_STRVAR(doc_imp, "(Extremely) low-level import machinery bits as used by importlib and imp."); -PyDoc_STRVAR(doc_get_magic, -"get_magic() -> string\n\ -Return the magic number for .pyc or .pyo files."); - PyDoc_STRVAR(doc_get_tag, "get_tag() -> string\n\ Return the magic tag for .pyc or .pyo files."); @@ -2219,7 +2114,6 @@ On platforms without threads, this function does nothing."); static PyMethodDef imp_methods[] = { - {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held},