comparing with /home/esnow/projects/.external/.cpython searching for changes changeset: 76471:f1c29fee826b tag: issue13959_magic.diff tag: qbase tag: qtip tag: tip user: Eric Snow date: Sun Apr 22 10:37:08 2012 +0000 summary: [issue 13959] port MAGIC and TAG to importlib diff --git a/Lib/imp.py b/Lib/imp.py --- a/Lib/imp.py +++ b/Lib/imp.py @@ -10,8 +10,6 @@ load_dynamic, get_frozen_object, is_frozen_package, init_builtin, init_frozen, is_builtin, is_frozen, _fix_co_filename) -# Could move out of _imp, but not worth the code -from _imp import get_magic, get_tag # Can (probably) move to importlib from _imp import get_suffixes # Should be re-implemented here (and mostly deprecated) @@ -40,6 +38,17 @@ IMP_HOOK = 9 +def get_magic(): + """Return the magic number for .pyc or .pyo files.""" + return _bootstrap._PYC_MAGIC_BYTES + + +def get_tag(): + """Return the magic tag for .pyc or .pyo files.""" + #return sys.implementation.pyc_tag + return _bootstrap._PYC_TAG + + def source_from_cache(path): """Given the path to a .pyc./.pyo file, return the path to its .py file. diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -168,6 +168,140 @@ code_type = type(_wrap.__code__) +# legacy imp interface support ################################################ + +# Magic word to reject .pyc files generated by other Python versions. +# It should change for each incompatible change to the bytecode. +# +# The value of CR and LF is incorporated so if you ever read or write +# a .pyc file in text mode the magic number will be wrong; also, the +# Apple MPW compiler swaps their values, botching string constants. +# +# The magic numbers must be spaced apart at least 2 values, as the +# -U interpeter flag will cause PYC_MAGIC+1 being used. They have been +# odd numbers for some time now. +# +# There were a variety of old schemes for setting the magic number. +# The current working scheme is to increment the previous value by +# 10. +# +# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic +# number also includes a new "magic tag", i.e. a human readable string used +# to represent the magic number in __pycache__ directories. When you change +# the magic number, you must also set a new unique magic tag. Generally this +# can be named after the Python major version of the magic number bump, but +# it can really be anything, as long as it's different than anything else +# that's come before. The tags are included in the following table, starting +# with Python 3.2a0. +# +# Known values: +# Python 1.5: 20121 +# Python 1.5.1: 20121 +# Python 1.5.2: 20121 +# Python 1.6: 50428 +# Python 2.0: 50823 +# Python 2.0.1: 50823 +# Python 2.1: 60202 +# Python 2.1.1: 60202 +# Python 2.1.2: 60202 +# Python 2.2: 60717 +# Python 2.3a0: 62011 +# Python 2.3a0: 62021 +# Python 2.3a0: 62011 (!) +# Python 2.4a0: 62041 +# Python 2.4a3: 62051 +# Python 2.4b1: 62061 +# Python 2.5a0: 62071 +# Python 2.5a0: 62081 (ast-branch) +# Python 2.5a0: 62091 (with) +# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) +# Python 2.5b3: 62101 (fix wrong code: for x, in ...) +# Python 2.5b3: 62111 (fix wrong code: x += yield) +# Python 2.5c1: 62121 (fix wrong lnotab with for loops and +# storing constants that should have been removed) +# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) +# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) +# Python 2.6a1: 62161 (WITH_CLEANUP optimization) +# Python 3000: 3000 +# 3010 (removed UNARY_CONVERT) +# 3020 (added BUILD_SET) +# 3030 (added keyword-only parameters) +# 3040 (added signature annotations) +# 3050 (print becomes a function) +# 3060 (PEP 3115 metaclass syntax) +# 3061 (string literals become unicode) +# 3071 (PEP 3109 raise changes) +# 3081 (PEP 3137 make __file__ and __name__ unicode) +# 3091 (kill str8 interning) +# 3101 (merge from 2.6a0, see 62151) +# 3103 (__file__ points to source file) +# Python 3.0a4: 3111 (WITH_CLEANUP optimization). +# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) +# Python 3.1a0: 3141 (optimize list, set and dict comprehensions: +# change LIST_APPEND and SET_ADD, add MAP_ADD) +# Python 3.1a0: 3151 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) +# Python 3.2a0: 3160 (add SETUP_WITH) +# tag: cpython-32 +# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) +# tag: cpython-32 +# Python 3.2a2 3180 (add DELETE_DEREF) +# Python 3.3a0 3190 __class__ super closure changed +# Python 3.3a0 3200 (__qualname__ added) +# 3210 (added size modulo 2**32 to the pyc header) +# Python 3.3a1 3220 (changed PEP 380 implementation) + +# PYC_MAGIC must change whenever the bytecode emitted by the compiler may no +# longer be understood by older implementations of the eval loop (usually +# due to the addition of new opcodes). +PYC_MAGIC = 3220 + +def _get_pyc_magic_int(magic_bytes, *, withlinesep=False): + """Return the magic number encoded in the bytes.""" + rawmagic = magic_bytes[0] + (magic_bytes[1] << 8) + if withlinesep: + rawmagic += magic_bytes[2] << 16 + rawmagic += magic_bytes[3] << 24 + return rawmagic + +def _get_pyc_magic_bytes(rawmagic=PYC_MAGIC, *, withlinesep=True, raw=True): + """Return the magic number for .pyc or .pyo files.""" + if withlinesep: + rawmagic |= (ord("\r") << 16) | (ord("\n") << 24) + if raw: + return rawmagic + return bytes([ + (rawmagic >> 0) & 0xff, + (rawmagic >> 8) & 0xff, + (rawmagic >> 16) & 0xff, + (rawmagic >> 24) & 0xff]) + +_PYC_MAGIC_BYTES = _get_pyc_magic_bytes(raw=False) + + +# PYC_TAG must change for each major Python release. The magic number +# will take care of any bytecode changes that occur during development. +# (See PEP 3147.) +def _get_tag(): + # first find the Python implementation + # XXX is there a better way? (sys.implementation.pyc_tag?) + # see http://mail.python.org/pipermail/python-ideas/2012-March/014555.html + if sys.version.startswith("IronPython"): + impl = 'ironpython' + elif sys.version.startswith("java"): + impl = 'jython' + elif "PyPy" in sys.version: + impl = 'pypy' + else: + impl = 'cpython' + + return "{}-{}{}".format(impl, *sys.version_info) + +# moved to _setup() due to reliance on sys +#PYC_TAG = _get_tag() +PYC_TAG = None + + def _new_module(name): """Create a new module. @@ -437,7 +571,7 @@ magic = data[:4] raw_timestamp = data[4:8] raw_size = data[8:12] - if magic != _MAGIC_NUMBER: + if magic != _PYC_MAGIC_BYTES: msg = 'bad magic number in {!r}: {!r}'.format(fullname, magic) raise ImportError(msg, name=fullname, path=bytecode_path) elif len(raw_timestamp) != 4: @@ -582,7 +716,7 @@ verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): - data = bytearray(_MAGIC_NUMBER) + data = bytearray(_PYC_MAGIC_BYTES) data.extend(_w_long(source_mtime)) data.extend(_w_long(len(source_bytes))) data.extend(marshal.dumps(code_object)) @@ -1148,10 +1282,6 @@ return _handle_fromlist(module, fromlist, _gcd_import) -_MAGIC_NUMBER = None # Set in _setup() -_TAG = None # Set in _setup() - - def _setup(sys_module, _imp_module): """Setup importlib by importing needed built-in modules and injecting them into the global namespace. @@ -1198,8 +1328,7 @@ setattr(self_module, 'path_separators', set(path_separators)) # Constants setattr(self_module, '_relax_case', _make_relax_case()) - setattr(self_module, '_MAGIC_NUMBER', _imp_module.get_magic()) - setattr(self_module, '_TAG', _imp.get_tag()) + setattr(self_module, '_PYC_TAG', _get_tag()) if builtin_os == 'nt': SOURCE_SUFFIXES.append('.pyw') diff --git a/Python/import.c b/Python/import.c --- a/Python/import.c +++ b/Python/import.c @@ -26,108 +26,9 @@ #endif -/* Magic word to reject .pyc files generated by other Python versions. - It should change for each incompatible change to the bytecode. +/* MAGIC has moved to Lib/importlib/_bootstrap.py, as PYC_MAGIC. */ - The value of CR and LF is incorporated so if you ever read or write - a .pyc file in text mode the magic number will be wrong; also, the - Apple MPW compiler swaps their values, botching string constants. - - The magic numbers must be spaced apart at least 2 values, as the - -U interpeter flag will cause MAGIC+1 being used. They have been - odd numbers for some time now. - - There were a variety of old schemes for setting the magic number. - The current working scheme is to increment the previous value by - 10. - - Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic - number also includes a new "magic tag", i.e. a human readable string used - to represent the magic number in __pycache__ directories. When you change - the magic number, you must also set a new unique magic tag. Generally this - can be named after the Python major version of the magic number bump, but - it can really be anything, as long as it's different than anything else - that's come before. The tags are included in the following table, starting - with Python 3.2a0. - - Known values: - Python 1.5: 20121 - Python 1.5.1: 20121 - Python 1.5.2: 20121 - Python 1.6: 50428 - Python 2.0: 50823 - Python 2.0.1: 50823 - Python 2.1: 60202 - Python 2.1.1: 60202 - Python 2.1.2: 60202 - Python 2.2: 60717 - Python 2.3a0: 62011 - Python 2.3a0: 62021 - Python 2.3a0: 62011 (!) - Python 2.4a0: 62041 - Python 2.4a3: 62051 - Python 2.4b1: 62061 - Python 2.5a0: 62071 - Python 2.5a0: 62081 (ast-branch) - Python 2.5a0: 62091 (with) - Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) - Python 2.5b3: 62101 (fix wrong code: for x, in ...) - Python 2.5b3: 62111 (fix wrong code: x += yield) - Python 2.5c1: 62121 (fix wrong lnotab with for loops and - storing constants that should have been removed) - Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) - Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) - Python 2.6a1: 62161 (WITH_CLEANUP optimization) - Python 3000: 3000 - 3010 (removed UNARY_CONVERT) - 3020 (added BUILD_SET) - 3030 (added keyword-only parameters) - 3040 (added signature annotations) - 3050 (print becomes a function) - 3060 (PEP 3115 metaclass syntax) - 3061 (string literals become unicode) - 3071 (PEP 3109 raise changes) - 3081 (PEP 3137 make __file__ and __name__ unicode) - 3091 (kill str8 interning) - 3101 (merge from 2.6a0, see 62151) - 3103 (__file__ points to source file) - Python 3.0a4: 3111 (WITH_CLEANUP optimization). - Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) - Python 3.1a0: 3141 (optimize list, set and dict comprehensions: - change LIST_APPEND and SET_ADD, add MAP_ADD) - Python 3.1a0: 3151 (optimize conditional branches: - introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) - Python 3.2a0: 3160 (add SETUP_WITH) - tag: cpython-32 - Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) - tag: cpython-32 - Python 3.2a2 3180 (add DELETE_DEREF) - Python 3.3a0 3190 __class__ super closure changed - Python 3.3a0 3200 (__qualname__ added) - 3210 (added size modulo 2**32 to the pyc header) - Python 3.3a1 3220 (changed PEP 380 implementation) -*/ - -/* MAGIC must change whenever the bytecode emitted by the compiler may no - longer be understood by older implementations of the eval loop (usually - due to the addition of new opcodes) - TAG must change for each major Python release. The magic number will take - care of any bytecode changes that occur during development. -*/ -#define QUOTE(arg) #arg -#define STRIFY(name) QUOTE(name) -#define MAJOR STRIFY(PY_MAJOR_VERSION) -#define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3220 | ((long)'\r'<<16) | ((long)'\n'<<24)) -#define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" -/* Current magic word and string tag as globals. */ -static long pyc_magic = MAGIC; -static const char *pyc_tag = TAG; -#undef QUOTE -#undef STRIFY -#undef MAJOR -#undef MINOR /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; @@ -588,16 +489,26 @@ long PyImport_GetMagicNumber(void) { - return pyc_magic; + PyInterpreterState *interp = PyThreadState_Get()->interp; + PyObject *pyc_magic = PyObject_CallMethod(interp->importlib, + "get_pyc_magic_bytes", NULL); + if (pyc_magic == NULL) + return NULL; + return PyLong_AsLong(pyc_magic); } const char * PyImport_GetMagicTag(void) { - return pyc_tag; + PyInterpreterState *interp = PyThreadState_Get()->interp; + PyObject *pyc_tag = PyObject_GetAttrString(interp->importlib, "_PYC_TAG"); + if (pyc_tag == NULL) + return NULL; + return _PyUnicode_AsString(pyc_tag); } + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by module name @@ -1970,31 +1881,6 @@ */ static PyObject * -imp_make_magic(long magic) -{ - char buf[4]; - - buf[0] = (char) ((magic >> 0) & 0xff); - buf[1] = (char) ((magic >> 8) & 0xff); - buf[2] = (char) ((magic >> 16) & 0xff); - buf[3] = (char) ((magic >> 24) & 0xff); - - return PyBytes_FromStringAndSize(buf, 4); -} - -static PyObject * -imp_get_magic(PyObject *self, PyObject *noargs) -{ - return imp_make_magic(pyc_magic); -} - -static PyObject * -imp_get_tag(PyObject *self, PyObject *noargs) -{ - return PyUnicode_FromString(pyc_tag); -} - -static PyObject * imp_get_suffixes(PyObject *self, PyObject *noargs) { PyObject *list; @@ -2188,14 +2074,6 @@ PyDoc_STRVAR(doc_imp, "(Extremely) low-level import machinery bits as used by importlib and imp."); -PyDoc_STRVAR(doc_get_magic, -"get_magic() -> string\n\ -Return the magic number for .pyc or .pyo files."); - -PyDoc_STRVAR(doc_get_tag, -"get_tag() -> string\n\ -Return the magic tag for .pyc or .pyo files."); - PyDoc_STRVAR(doc_get_suffixes, "get_suffixes() -> [(suffix, mode, type), ...]\n\ Return a list of (suffix, mode, type) tuples describing the files\n\ @@ -2219,8 +2097,6 @@ On platforms without threads, this function does nothing."); static PyMethodDef imp_methods[] = { - {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, - {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock},