diff -r e281a57d5b29 Include/dictobject.h --- a/Include/dictobject.h Tue Apr 19 08:53:14 2016 +0200 +++ b/Include/dictobject.h Tue Apr 19 12:20:08 2016 +0200 @@ -23,6 +23,7 @@ typedef struct _dictkeysobject PyDictKey typedef struct { PyObject_HEAD Py_ssize_t ma_used; + PY_UINT64_T ma_version_tag; PyDictKeysObject *ma_keys; PyObject **ma_values; } PyDictObject; diff -r e281a57d5b29 Lib/test/test_ordered_dict.py --- a/Lib/test/test_ordered_dict.py Tue Apr 19 08:53:14 2016 +0200 +++ b/Lib/test/test_ordered_dict.py Tue Apr 19 12:20:08 2016 +0200 @@ -635,7 +635,7 @@ class CPythonOrderedDictTests(OrderedDic size = support.calcobjsize check = self.check_sizeof - basicsize = size('n2P' + '3PnPn2P') + calcsize('2nPn') + basicsize = size('nL2P' + '3PnPn2P') + calcsize('2nPn') entrysize = calcsize('n2P') + calcsize('P') nodesize = calcsize('Pn2P') diff -r e281a57d5b29 Lib/test/test_pep509.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/test_pep509.py Tue Apr 19 12:20:08 2016 +0200 @@ -0,0 +1,186 @@ +""" +Test implementation of the PEP 509: dictionary versionning. +""" +import unittest +from test import support + +# PEP 509 is implemented in CPython but other Python implementations +# don't require to implement it +_testcapi = support.import_module('_testcapi') + + +class DictVersionTests(unittest.TestCase): + type2test = dict + + def setUp(self): + self.seen_versions = set() + self.dict = None + + def check_version_unique(self, mydict): + version = _testcapi.dict_get_version(mydict) + self.assertNotIn(version, self.seen_versions) + self.seen_versions.add(version) + + def check_version_changed(self, mydict, method, *args, **kw): + result = method(*args, **kw) + self.check_version_unique(mydict) + return result + + def check_version_dont_change(self, mydict, method, *args, **kw): + version1 = _testcapi.dict_get_version(mydict) + self.seen_versions.add(version1) + + result = method(*args, **kw) + + version2 = _testcapi.dict_get_version(mydict) + self.assertEqual(version2, version1, "version changed") + + return result + + def new_dict(self, *args, **kw): + d = self.type2test(*args, **kw) + self.check_version_unique(d) + return d + + def test_constructor(self): + # new empty dictionaries must all have an unique version + empty1 = self.new_dict() + empty2 = self.new_dict() + empty3 = self.new_dict() + + # non-empty dictionaries must also have an unique version + nonempty1 = self.new_dict(x='x') + nonempty2 = self.new_dict(x='x', y='y') + + def test_copy(self): + d = self.new_dict(a=1, b=2) + + d2 = self.check_version_dont_change(d, d.copy) + + # dict.copy() must create a dictionary with a new unique version + self.check_version_unique(d2) + + def test_setitem(self): + d = self.new_dict() + + # creating new keys must change the version + self.check_version_changed(d, d.__setitem__, 'x', 'x') + self.check_version_changed(d, d.__setitem__, 'y', 'y') + + # changing values must change the version + self.check_version_changed(d, d.__setitem__, 'x', 1) + self.check_version_changed(d, d.__setitem__, 'y', 2) + + def test_setitem_same_value(self): + value = object() + d = self.new_dict() + + # setting a key must change the version + self.check_version_changed(d, d.__setitem__, 'key', value) + + # setting a key to the same value with dict.__setitem__ + # must change the version + self.check_version_changed(d, d.__setitem__, 'key', value) + + # setting a key to the same value with dict.update + # must change the version + self.check_version_changed(d, d.update, key=value) + + d2 = self.new_dict(key=value) + self.check_version_changed(d, d.update, d2) + + def test_setitem_equal(self): + class AlwaysEqual: + def __eq__(self, other): + return True + + value1 = AlwaysEqual() + value2 = AlwaysEqual() + self.assertTrue(value1 == value2) + self.assertFalse(value1 != value2) + + d = self.new_dict() + self.check_version_changed(d, d.__setitem__, 'key', value1) + + # setting a key to a value equal to the current value + # with dict.__setitem__() must change the version + self.check_version_changed(d, d.__setitem__, 'key', value2) + + # setting a key to a value equal to the current value + # with dict.update() must change the version + self.check_version_changed(d, d.update, key=value1) + + d2 = self.new_dict(key=value2) + self.check_version_changed(d, d.update, d2) + + def test_setdefault(self): + d = self.new_dict() + + # setting a key with dict.setdefault() must change the version + self.check_version_changed(d, d.setdefault, 'key', 'value1') + + # don't change the version if the key already exists + self.check_version_dont_change(d, d.setdefault, 'key', 'value2') + + def test_delitem(self): + d = self.new_dict(key='value') + + # deleting a key with dict.__delitem__() must change the version + self.check_version_changed(d, d.__delitem__, 'key') + + # don't change the version if the key doesn't exist + self.check_version_dont_change(d, self.assertRaises, KeyError, + d.__delitem__, 'key') + + def test_pop(self): + d = self.new_dict(key='value') + + # pop() must change the version if the key exists + self.check_version_changed(d, d.pop, 'key') + + # pop() must not change the version if the key does not exist + self.check_version_dont_change(d, self.assertRaises, KeyError, + d.pop, 'key') + + def test_popitem(self): + d = self.new_dict(key='value') + + # popitem() must change the version if the dict is not empty + self.check_version_changed(d, d.popitem) + + # popitem() must not change the version if the dict is empty + self.check_version_dont_change(d, self.assertRaises, KeyError, + d.popitem) + + def test_update(self): + d = self.new_dict(key='value') + + # update() calling with no argument must not change the version + self.check_version_dont_change(d, d.update) + + # update() must change the version + self.check_version_changed(d, d.update, key='new value') + + d2 = self.new_dict(key='value 3') + self.check_version_changed(d, d.update, d2) + + def test_clear(self): + d = self.new_dict(key='value') + + # clear() must change the version if the dict is not empty + self.check_version_changed(d, d.clear) + + # clear() must not change the version if the dict is empty + self.check_version_dont_change(d, d.clear) + + +class Dict(dict): + pass + + +class DictSubtypeVersionTests(DictVersionTests): + type2test = Dict + + +if __name__ == "__main__": + unittest.main() diff -r e281a57d5b29 Lib/test/test_sys.py --- a/Lib/test/test_sys.py Tue Apr 19 08:53:14 2016 +0200 +++ b/Lib/test/test_sys.py Tue Apr 19 12:20:08 2016 +0200 @@ -935,9 +935,9 @@ class SizeofTest(unittest.TestCase): # method-wrapper (descriptor object) check({}.__iter__, size('2P')) # dict - check({}, size('n2P') + calcsize('2nPn') + 8*calcsize('n2P')) + check({}, size('n2P') + 8 + calcsize('2nPn') + 8*calcsize('n2P')) longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8} - check(longdict, size('n2P') + calcsize('2nPn') + 16*calcsize('n2P')) + check(longdict, size('n2P') + 8 + calcsize('2nPn') + 16*calcsize('n2P')) # dictionary-keyview check({}.keys(), size('P')) # dictionary-valueview @@ -1098,7 +1098,7 @@ class SizeofTest(unittest.TestCase): class newstyleclass(object): pass check(newstyleclass, s) # dict with shared keys - check(newstyleclass().__dict__, size('n2P' + '2nPn')) + check(newstyleclass().__dict__, size('n2P' + '2nPn') + 8) # unicode # each tuple contains a string and its expected character size # don't put any static strings here, as they may contain diff -r e281a57d5b29 Modules/_testcapimodule.c --- a/Modules/_testcapimodule.c Tue Apr 19 08:53:14 2016 +0200 +++ b/Modules/_testcapimodule.c Tue Apr 19 12:20:08 2016 +0200 @@ -3747,6 +3747,26 @@ tracemalloc_get_traceback(PyObject *self return _PyTraceMalloc_GetTraceback(domain, (Py_uintptr_t)ptr); } +static PyObject * +dict_get_version(PyObject *self, PyObject *args) +{ + PyDictObject *dict; + PY_UINT64_T version; + + if (!PyArg_ParseTuple(args, "O!", &PyDict_Type, &dict)) + return NULL; + + version = dict->ma_version_tag; + +#ifdef HAVE_LONG_LONG + Py_BUILD_ASSERT(sizeof(unsigned PY_LONG_LONG) >= sizeof(version)); + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)version); +#else + Py_BUILD_ASSERT(sizeof(size_t) >= sizeof(version)); + return PyLong_FromSize_t((size_t)version); +#endif +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3936,6 +3956,7 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, + {"dict_get_version", dict_get_version, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff -r e281a57d5b29 Objects/dictobject.c --- a/Objects/dictobject.c Tue Apr 19 08:53:14 2016 +0200 +++ b/Objects/dictobject.c Tue Apr 19 12:20:08 2016 +0200 @@ -209,6 +209,13 @@ static PyDictKeyEntry *lookdict_split(Py static int dictresize(PyDictObject *mp, Py_ssize_t minused); +/* Global counter used to set ma_version_tag field of dictionary. + * It is incremented each time that a dictionary is created and each + * time that a dictionary is modified. */ +static PY_UINT64_T pydict_global_version = 0; + +#define DICT_NEXT_VERSION() (++pydict_global_version ) + /* Dictionary reuse scheme to save calls to malloc, free, and memset */ #ifndef PyDict_MAXFREELIST #define PyDict_MAXFREELIST 80 @@ -383,6 +390,7 @@ new_dict(PyDictKeysObject *keys, PyObjec mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = 0; + mp->ma_version_tag = DICT_NEXT_VERSION(); return (PyObject *)mp; } @@ -802,13 +810,18 @@ insertdict(PyDictObject *mp, PyObject *k assert(PyUnicode_CheckExact(key) || mp->ma_keys->dk_lookup == lookdict); Py_INCREF(value); MAINTAIN_TRACKING(mp, key, value); + old_value = *value_addr; if (old_value != NULL) { assert(ep->me_key != NULL && ep->me_key != dummy); *value_addr = value; + mp->ma_version_tag = DICT_NEXT_VERSION(); + Py_DECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ } else { + mp->ma_version_tag = DICT_NEXT_VERSION(); + if (ep->me_key == NULL) { Py_INCREF(key); if (mp->ma_keys->dk_usable <= 0) { @@ -1278,6 +1291,8 @@ PyDict_DelItem(PyObject *op, PyObject *k _PyErr_SetKeyError(key); return -1; } + + mp->ma_version_tag = DICT_NEXT_VERSION(); old_value = *value_addr; *value_addr = NULL; mp->ma_used--; @@ -1348,6 +1363,7 @@ PyDict_Clear(PyObject *op) mp->ma_keys = Py_EMPTY_KEYS; mp->ma_values = empty_values; mp->ma_used = 0; + mp->ma_version_tag = DICT_NEXT_VERSION(); /* ...then clear the keys and values */ if (oldvalues != NULL) { n = DK_SIZE(oldkeys); @@ -1481,8 +1497,11 @@ PyObject * _PyErr_SetKeyError(key); return NULL; } + *value_addr = NULL; + mp->ma_version_tag = DICT_NEXT_VERSION(); mp->ma_used--; + if (!_PyDict_HasSplitTable(mp)) { ENSURE_ALLOWS_DELETIONS(mp); old_key = ep->me_key; @@ -2415,6 +2434,7 @@ PyDict_SetDefault(PyObject *d, PyObject val = defaultobj; mp->ma_keys->dk_usable--; mp->ma_used++; + mp->ma_version_tag = DICT_NEXT_VERSION(); } return val; } @@ -2513,6 +2533,7 @@ dict_popitem(PyDictObject *mp) Py_INCREF(dummy); ep->me_key = dummy; ep->me_value = NULL; + mp->ma_version_tag = DICT_NEXT_VERSION(); mp->ma_used--; assert(mp->ma_keys->dk_entries[0].me_value == NULL); mp->ma_keys->dk_entries[0].me_hash = i + 1; /* next place to start */ @@ -2718,6 +2739,7 @@ dict_new(PyTypeObject *type, PyObject *a _PyObject_GC_UNTRACK(d); d->ma_used = 0; + d->ma_version_tag = DICT_NEXT_VERSION(); d->ma_keys = new_keys_object(PyDict_MINSIZE_COMBINED); if (d->ma_keys == NULL) { Py_DECREF(self);