From 3993d606c061dc064c340bbf2cf964cc0d0ca76e Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 3 Feb 2017 11:55:57 +1300 Subject: [PATCH] Add a PYTHONREVERSEDICTKEYORDER environment variable Due to implementation changes, since CPython 3.6 dict keys are returned in insertion order. However, in order to test for reproducible builds [0], it would be convenient to be able to reverse this ordering; we would then run a build of an arbitrary package both with and without this flag and compare the resulting binary. (We already run such a testing framework, so specifying this environment variable would be trivial. Note that this "reverse" would actually find more issues than simply relying on the pre-3.6 non-deterministic behaviour.) This patch changes the behaviour of: * for x in d: * d.popitem() * d.items() * _PyDict_Next [0] https://reproducible-builds.org/ Signed-off-by: Chris Lamb --- Doc/using/cmdline.rst | 7 +++++++ Doc/whatsnew/3.7.rst | 10 +++++++++ Include/pydebug.h | 1 + Misc/python.man | 3 +++ Modules/main.c | 3 ++- Objects/dictobject.c | 56 ++++++++++++++++++++++++++++++++++----------------- Python/pylifecycle.c | 3 +++ 7 files changed, 64 insertions(+), 19 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index c0e64d66f2..c3e639645a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -535,6 +535,13 @@ conflict. specifying the :option:`-B` option. +.. envvar:: PYTHONREVERSEDICTKEYORDER + + If this is set to a non-empty string, Python will reverse the order that + dict keys/items are returned. This is to help test for `Reproducible Builds + `_. + + .. envvar:: PYTHONHASHSEED If this variable is not set or set to ``random``, a random value is used diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 81ad4f9245..1f5ff83272 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -70,6 +70,16 @@ Summary -- Release highlights New Features ============ +* As part of the changes to use a :ref:`more compact representation + ` in Python 3.6, :ref:`dict ` keys are + returned in insertion order. However, in order to test for `Reproducible + Builds `_ it is convenient to be able to + reverse this ordering - testers can then run a build of an arbitrary Python + package both with and without the keys reversed and compare the results. + + To enable this behaviour, set the :envvar:`PYTHONREVERSEDICTKEYORDER` + environment variable before running the interpreter. (Contributed by Chris + Lamb) Other Language Changes diff --git a/Include/pydebug.h b/Include/pydebug.h index 6e23a896c3..ad1d310e0b 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -19,6 +19,7 @@ PyAPI_DATA(int) Py_UseClassExceptionsFlag; PyAPI_DATA(int) Py_FrozenFlag; PyAPI_DATA(int) Py_IgnoreEnvironmentFlag; PyAPI_DATA(int) Py_DontWriteBytecodeFlag; +PyAPI_DATA(int) Py_ReverseDictKeyOrderFlag; PyAPI_DATA(int) Py_NoUserSiteDirectory; PyAPI_DATA(int) Py_UnbufferedStdioFlag; PyAPI_DATA(int) Py_HashRandomizationFlag; diff --git a/Misc/python.man b/Misc/python.man index 385b6546c8..69c6f691f7 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -394,6 +394,9 @@ If this is set to a non-empty string it is equivalent to specifying the \fB\-B\fP option (don't try to write .I .pyc files). +.IP PYTHONREVERSEDICTKEYORDER +If this is set to a non-empty string, reverse the order of that dict keys and +items are returned. This is to help test for reproducible builds. .IP PYTHONINSPECT If this is set to a non-empty string it is equivalent to specifying the \fB\-i\fP option. diff --git a/Modules/main.c b/Modules/main.c index 020c353e0c..a28d2cad11 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -105,7 +105,8 @@ static const char usage_6[] = " predictable seed.\n" "PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" " on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" -" hooks.\n"; +" hooks.\n" +"PYTHONREVERSEDICTKEYORDER: reverse ordering of dict keys, items, etc.\n"; static int usage(int exitcode, const wchar_t* program) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 00fd58c81d..ffe4e40d2f 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2984,11 +2984,19 @@ dict_popitem(PyDictObject *mp) } ENSURE_ALLOWS_DELETIONS(mp); - /* Pop last item */ ep0 = DK_ENTRIES(mp->ma_keys); - i = mp->ma_keys->dk_nentries - 1; - while (i >= 0 && ep0[i].me_value == NULL) { - i--; + if (Py_ReverseDictKeyOrderFlag) { + /* Pop first item */ + i = 0; + while (i < mp->ma_keys->dk_nentries && ep0[i].me_value == NULL) { + i++; + } + } else { + /* Pop last item */ + i = mp->ma_keys->dk_nentries - 1; + while (i >= 0 && ep0[i].me_value == NULL) { + i--; + } } assert(i >= 0); @@ -3448,7 +3456,7 @@ static PyObject* dictiter_iternextkey(dictiterobject *di) { PyObject *key; - Py_ssize_t i; + Py_ssize_t i, idx; PyDictKeysObject *k; PyDictObject *d = di->di_dict; @@ -3465,18 +3473,22 @@ dictiter_iternextkey(dictiterobject *di) i = di->di_pos; k = d->ma_keys; + idx = Py_ReverseDictKeyOrderFlag ? (k->dk_nentries - i - 1) : i; assert(i >= 0); if (d->ma_values) { if (i >= d->ma_used) goto fail; - key = DK_ENTRIES(k)[i].me_key; - assert(d->ma_values[i] != NULL); + key = DK_ENTRIES(k)[idx].me_key; + assert(d->ma_values[idx] != NULL); } else { Py_ssize_t n = k->dk_nentries; - PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; + PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[idx]; while (i < n && entry_ptr->me_value == NULL) { - entry_ptr++; + if (Py_ReverseDictKeyOrderFlag) + entry_ptr--; + else + entry_ptr++; i++; } if (i >= n) @@ -3531,7 +3543,7 @@ static PyObject * dictiter_iternextvalue(dictiterobject *di) { PyObject *value; - Py_ssize_t i; + Py_ssize_t i, idx; PyDictObject *d = di->di_dict; if (d == NULL) @@ -3546,18 +3558,22 @@ dictiter_iternextvalue(dictiterobject *di) } i = di->di_pos; + idx = Py_ReverseDictKeyOrderFlag ? (d->ma_keys->dk_nentries - i - 1) : i; assert(i >= 0); if (d->ma_values) { if (i >= d->ma_used) goto fail; - value = d->ma_values[i]; + value = d->ma_values[idx]; assert(value != NULL); } else { Py_ssize_t n = d->ma_keys->dk_nentries; - PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[i]; + PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[idx]; while (i < n && entry_ptr->me_value == NULL) { - entry_ptr++; + if (Py_ReverseDictKeyOrderFlag) + entry_ptr--; + else + entry_ptr++; i++; } if (i >= n) @@ -3612,7 +3628,7 @@ static PyObject * dictiter_iternextitem(dictiterobject *di) { PyObject *key, *value, *result = di->di_result; - Py_ssize_t i; + Py_ssize_t i, idx; PyDictObject *d = di->di_dict; if (d == NULL) @@ -3627,19 +3643,23 @@ dictiter_iternextitem(dictiterobject *di) } i = di->di_pos; + idx = Py_ReverseDictKeyOrderFlag ? (d->ma_keys->dk_nentries - i - 1) : i; assert(i >= 0); if (d->ma_values) { if (i >= d->ma_used) goto fail; - key = DK_ENTRIES(d->ma_keys)[i].me_key; - value = d->ma_values[i]; + key = DK_ENTRIES(d->ma_keys)[idx].me_key; + value = d->ma_values[idx]; assert(value != NULL); } else { Py_ssize_t n = d->ma_keys->dk_nentries; - PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[i]; + PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[idx]; while (i < n && entry_ptr->me_value == NULL) { - entry_ptr++; + if (Py_ReverseDictKeyOrderFlag) + entry_ptr--; + else + entry_ptr++; i++; } if (i >= n) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 06030c330a..a0eb0c9989 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -89,6 +89,7 @@ int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ +int Py_ReverseDictKeyOrderFlag = 0; /* Reverse dict items/key ordering */ int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ @@ -331,6 +332,8 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) Py_OptimizeFlag = add_flag(Py_OptimizeFlag, p); if ((p = Py_GETENV("PYTHONDONTWRITEBYTECODE")) && *p != '\0') Py_DontWriteBytecodeFlag = add_flag(Py_DontWriteBytecodeFlag, p); + if ((p = Py_GETENV("PYTHONREVERSEDICTKEYORDER")) && *p != '\0') + Py_ReverseDictKeyOrderFlag = add_flag(Py_ReverseDictKeyOrderFlag, p); /* The variable is only tested for existence here; _PyRandom_Init will check its value further. */ if ((p = Py_GETENV("PYTHONHASHSEED")) && *p != '\0') -- 2.11.0