diff -r 34986f55a32d Include/Python.h --- a/Include/Python.h Sun Oct 20 17:25:34 2013 +0300 +++ b/Include/Python.h Sun Oct 20 21:31:20 2013 +0300 @@ -84,6 +84,7 @@ #include "tupleobject.h" #include "listobject.h" #include "dictobject.h" +#include "odictobject.h" #include "enumobject.h" #include "setobject.h" #include "methodobject.h" diff -r 34986f55a32d Include/dictobject.h --- a/Include/dictobject.h Sun Oct 20 17:25:34 2013 +0300 +++ b/Include/dictobject.h Sun Oct 20 21:31:20 2013 +0300 @@ -27,6 +27,11 @@ PyObject **ma_values; } PyDictObject; +typedef struct { + PyObject_HEAD + PyDictObject *dv_dict; +} _PyDictViewObject; + #endif /* Py_LIMITED_API */ PyAPI_DATA(PyTypeObject) PyDict_Type; @@ -65,6 +70,7 @@ PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); PyAPI_FUNC(int) _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); +PyObject *_PyDictView_New(PyObject *, PyTypeObject *); #endif PyAPI_FUNC(PyObject *) PyDict_Keys(PyObject *mp); PyAPI_FUNC(PyObject *) PyDict_Values(PyObject *mp); @@ -78,6 +84,9 @@ PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp); PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp); Py_ssize_t _PyDict_KeysSize(PyDictKeysObject *keys); +PyObject *_PyDict_SizeOf(PyDictObject *); +PyObject *_PyDict_Pop(PyDictObject *, PyObject *, PyObject *); +PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) PyAPI_FUNC(int) PyDict_ClearFreeList(void); diff -r 34986f55a32d Include/odictobject.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Include/odictobject.h Sun Oct 20 21:31:20 2013 +0300 @@ -0,0 +1,45 @@ +#ifndef Py_ODICTOBJECT_H +#define Py_ODICTOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* OrderedDict */ + +#ifndef Py_LIMITED_API + +typedef struct _odictobject PyODictObject; + +PyAPI_DATA(PyTypeObject) PyODict_Type; +PyAPI_DATA(PyTypeObject) PyODictIterKey_Type; +PyAPI_DATA(PyTypeObject) PyODictIterItem_Type; +PyAPI_DATA(PyTypeObject) PyODictIterValue_Type; +PyAPI_DATA(PyTypeObject) PyODictIterReversed_Type; +PyAPI_DATA(PyTypeObject) PyODictKeys_Type; +PyAPI_DATA(PyTypeObject) PyODictItems_Type; +PyAPI_DATA(PyTypeObject) PyODictValues_Type; + +#endif /* Py_LIMITED_API */ + +#define PyODict_Check(op) PyObject_IsInstance(op, (PyObject *)&PyODict_Type) +#define PyODict_CheckExact(op) (Py_TYPE(op) == &PyODict_Type) +#define PyODict_SIZE(op) ((PyDictObject *)op)->ma_used +#define PyODict_HasKey(od, key) (PyMapping_HasKey(PyObject *)od, key) + +PyAPI_FUNC(PyObject *) PyODict_New(void); +PyAPI_FUNC(int) PyODict_SetItem(PyObject *od, PyObject *key, PyObject *item); +PyAPI_FUNC(int) PyODict_DelItem(PyObject *od, PyObject *key); + +/* wrappers around PyDict* functions */ +#define PyODict_GetItem(od, key) PyDict_GetItem((PyObject *)od, key) +#define PyODict_Contains(od, key) PyDict_Contains((PyObject *)od, key) +#define PyODict_Size(od) PyDict_Size((PyObject *)od) +#define PyODict_GetItemString(od, key) \ + PyDict_GetItemString((PyObject *)od, key) +#define Py_ODict_GetItemId(od, key) _PyDict_GetItemId((PyObject *)od, key) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_ODICTOBJECT_H */ diff -r 34986f55a32d Lib/collections/__init__.py --- a/Lib/collections/__init__.py Sun Oct 20 17:25:34 2013 +0300 +++ b/Lib/collections/__init__.py Sun Oct 20 21:31:20 2013 +0300 @@ -7,7 +7,6 @@ import _collections_abc __all__ += _collections_abc.__all__ -from _collections import deque, defaultdict from operator import itemgetter as _itemgetter, eq as _eq from keyword import iskeyword as _iskeyword import sys as _sys @@ -16,6 +15,17 @@ from itertools import repeat as _repeat, chain as _chain, starmap as _starmap from reprlib import recursive_repr as _recursive_repr +try: + from _collections import deque +except ImportError: + pass + +try: + from _collections import defaultdict +except ImportError: + pass + + ################################################################################ ### OrderedDict ################################################################################ @@ -44,6 +54,10 @@ their insertion order is arbitrary. ''' +# import sys +# frame = sys._getframe(1) +# print("{}, line {}".format(frame.f_globals['__file__'], frame.f_lineno)) +# raise Exception if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) try: @@ -60,6 +74,7 @@ 'od.__setitem__(i, y) <==> od[i]=y' # Setting a new item creates a new link at the end of the linked list, # and the inherited dictionary is updated with the new key/value pair. +# raise Exception if key not in self: self.__map[key] = link = Link() root = self.__root @@ -229,6 +244,12 @@ return dict.__eq__(self, other) +try: + from _collections import OrderedDict +except ImportError: + pass + + ################################################################################ ### namedtuple ################################################################################ diff -r 34986f55a32d Lib/test/test_collections.py --- a/Lib/test/test_collections.py Sun Oct 20 17:25:34 2013 +0300 +++ b/Lib/test/test_collections.py Sun Oct 20 21:31:20 2013 +0300 @@ -1,7 +1,8 @@ """Unit tests for collections.py.""" import unittest, doctest, operator -from test.support import TESTFN, forget, unlink +from test.support import TESTFN, forget, unlink, import_fresh_module +import contextlib import inspect from test import support from collections import namedtuple, Counter, OrderedDict, _count_elements @@ -1090,9 +1091,24 @@ ### OrderedDict ################################################################################ -class TestOrderedDict(unittest.TestCase): +py_coll = import_fresh_module('collections', blocked=['_collections']) +c_coll = import_fresh_module('collections', fresh=['_collections']) + + +@contextlib.contextmanager +def replaced_module(name, replacement): + original_module = sys.modules[name] + sys.modules[name] = replacement + try: + yield + finally: + sys.modules[name] = original_module + + +class OrderedDictTests: def test_init(self): + OrderedDict = self.module.OrderedDict with self.assertRaises(TypeError): OrderedDict([('a', 1), ('b', 2)], None) # too many args pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)] @@ -1102,17 +1118,22 @@ self.assertEqual(list(OrderedDict([('a', 1), ('b', 2), ('c', 9), ('d', 4)], c=3, e=5).items()), pairs) # mixed input - # make sure no positional args conflict with possible kwdargs - self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args, - ['self']) - # Make sure that direct calls to __init__ do not clear previous contents d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)]) d.__init__([('e', 5), ('f', 6)], g=7, d=4) self.assertEqual(list(d.items()), [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6), ('g', 7)]) + def test_init_argspec(self): + OrderedDict = self.module.OrderedDict + if self.module == c_coll: + raise unittest.SkipTest("only valid for pure Python") + # make sure no positional args conflict with possible kwdargs + self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args, + ['self']) + def test_update(self): + OrderedDict = self.module.OrderedDict with self.assertRaises(TypeError): OrderedDict().update([('a', 1), ('b', 2)], None) # too many args pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)] @@ -1150,10 +1171,12 @@ [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6), ('g', 7)]) def test_abc(self): + OrderedDict = self.module.OrderedDict self.assertIsInstance(OrderedDict(), MutableMapping) self.assertTrue(issubclass(OrderedDict, MutableMapping)) def test_clear(self): + OrderedDict = self.module.OrderedDict pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od = OrderedDict(pairs) @@ -1162,6 +1185,7 @@ self.assertEqual(len(od), 0) def test_delitem(self): + OrderedDict = self.module.OrderedDict pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) del od['a'] @@ -1171,6 +1195,7 @@ self.assertEqual(list(od.items()), pairs[:2] + pairs[3:]) def test_setitem(self): + OrderedDict = self.module.OrderedDict od = OrderedDict([('d', 1), ('b', 2), ('c', 3), ('a', 4), ('e', 5)]) od['c'] = 10 # existing element od['f'] = 20 # new element @@ -1178,6 +1203,7 @@ [('d', 1), ('b', 2), ('c', 10), ('a', 4), ('e', 5), ('f', 20)]) def test_iterators(self): + OrderedDict = self.module.OrderedDict pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od = OrderedDict(pairs) @@ -1188,7 +1214,21 @@ self.assertEqual(list(reversed(od)), [t[0] for t in reversed(pairs)]) + def test_sorted_iterators(self): + OrderedDict = self.module.OrderedDict + with self.assertRaises(TypeError): + OrderedDict([('a', 1), ('b', 2)], None) + pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)] + od = OrderedDict(pairs) + self.assertEqual(sorted(od), [t[0] for t in pairs]) + self.assertEqual(sorted(od.keys()), [t[0] for t in pairs]) + self.assertEqual(sorted(od.values()), [t[1] for t in pairs]) + self.assertEqual(sorted(od.items()), pairs) + self.assertEqual(sorted(reversed(od)), + sorted([t[0] for t in reversed(pairs)])) + def test_popitem(self): + OrderedDict = self.module.OrderedDict pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od = OrderedDict(pairs) @@ -1198,7 +1238,18 @@ od.popitem() self.assertEqual(len(od), 0) + def test_popitem_last(self): + OrderedDict = self.module.OrderedDict + pairs = [(i, i) for i in range(30)] + + obj = OrderedDict(pairs) + for i in range(8): + obj.popitem(True) + obj.popitem(True) + self.assertEqual(len(obj), 21) + def test_pop(self): + OrderedDict = self.module.OrderedDict pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od = OrderedDict(pairs) @@ -1223,6 +1274,7 @@ m.pop('a') def test_equality(self): + OrderedDict = self.module.OrderedDict pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od1 = OrderedDict(pairs) @@ -1238,13 +1290,15 @@ self.assertNotEqual(od1, OrderedDict(pairs[:-1])) def test_copying(self): + OrderedDict = self.module.OrderedDict # Check that ordered dicts are copyable, deepcopyable, picklable, # and have a repr/eval round-trip pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) update_test = OrderedDict() update_test.update(od) - for label, dup in [ + with replaced_module('collections', self.module): + for label, dup in [ ('od.copy()', od.copy()), ('copy.copy(od)', copy.copy(od)), ('copy.deepcopy(od)', copy.deepcopy(od)), @@ -1262,10 +1316,10 @@ ('update_test', update_test), ('OrderedDict(od)', OrderedDict(od)), ]: - with self.subTest(label=label): - msg = "\ncopy: %s\nod: %s" % (dup, od) - self.assertIsNot(dup, od, msg) - self.assertEqual(dup, od) + with self.subTest(label=label): + msg = "\ncopy: %s\nod: %s" % (dup, od) + self.assertIsNot(dup, od, msg) + self.assertEqual(dup, od) def test_yaml_linkage(self): # Verify that __reduce__ is setup in a way that supports PyYAML's dump() feature. @@ -1277,6 +1331,7 @@ self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1])) def test_reduce_not_too_fat(self): + OrderedDict = self.module.OrderedDict # do not save instance dictionary if not needed pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) @@ -1285,15 +1340,20 @@ self.assertIsNotNone(od.__reduce__()[2]) def test_pickle_recursive(self): + OrderedDict = self.module.OrderedDict od = OrderedDict() od[1] = od - for proto in range(-1, pickle.HIGHEST_PROTOCOL + 1): - dup = pickle.loads(pickle.dumps(od, proto)) - self.assertIsNot(dup, od) - self.assertEqual(list(dup.keys()), [1]) - self.assertIs(dup[1], dup) + + # pickle directly pulls the module, so we have to fake it + with replaced_module('collections', self.module): + for proto in range(-1, pickle.HIGHEST_PROTOCOL + 1): + dup = pickle.loads(pickle.dumps(od, proto)) + self.assertIsNot(dup, od) + self.assertEqual(list(dup.keys()), [1]) + self.assertIs(dup[1], dup) def test_repr(self): + OrderedDict = self.module.OrderedDict od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]) self.assertEqual(repr(od), "OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)])") @@ -1301,6 +1361,7 @@ self.assertEqual(repr(OrderedDict()), "OrderedDict()") def test_repr_recursive(self): + OrderedDict = self.module.OrderedDict # See issue #9826 od = OrderedDict.fromkeys('abc') od['x'] = od @@ -1308,6 +1369,7 @@ "OrderedDict([('a', None), ('b', None), ('c', None), ('x', ...)])") def test_setdefault(self): + OrderedDict = self.module.OrderedDict pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od = OrderedDict(pairs) @@ -1326,16 +1388,19 @@ self.assertEqual(Missing().setdefault(5, 9), 9) def test_reinsert(self): + OrderedDict = self.module.OrderedDict # Given insert a, insert b, delete a, re-insert a, # verify that a is now later than b. od = OrderedDict() od['a'] = 1 od['b'] = 2 del od['a'] + self.assertEqual(list(od.items()), [('b', 2)]) od['a'] = 1 self.assertEqual(list(od.items()), [('b', 2), ('a', 1)]) def test_move_to_end(self): + OrderedDict = self.module.OrderedDict od = OrderedDict.fromkeys('abcde') self.assertEqual(list(od), list('abcde')) od.move_to_end('c') @@ -1350,12 +1415,14 @@ od.move_to_end('x') def test_sizeof(self): + OrderedDict = self.module.OrderedDict # Wimpy test: Just verify the reported size is larger than a regular dict d = dict(a=1) od = OrderedDict(**d) self.assertGreater(sys.getsizeof(od), sys.getsizeof(d)) def test_override_update(self): + OrderedDict = self.module.OrderedDict # Verify that subclasses can override update() without breaking __init__() class MyOD(OrderedDict): def update(self, *args, **kwds): @@ -1363,18 +1430,62 @@ items = [('a', 1), ('c', 3), ('b', 2)] self.assertEqual(list(MyOD(items).items()), items) -class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol): - type2test = OrderedDict + +class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase): + + module = py_coll + + +@unittest.skipUnless(c_coll, 'requires the C version of the collections module') +class CPythonOrderedDictTests(OrderedDictTests, unittest.TestCase): + + module = c_coll + + +class PurePythonGeneralMappingTests(mapping_tests.BasicTestMappingProtocol): + + @classmethod + def setUpClass(cls): + cls.type2test = py_coll.OrderedDict def test_popitem(self): d = self._empty_mapping() self.assertRaises(KeyError, d.popitem) -class MyOrderedDict(OrderedDict): - pass -class SubclassMappingTests(mapping_tests.BasicTestMappingProtocol): - type2test = MyOrderedDict +@unittest.skipUnless(c_coll, 'requires the C version of the collections module') +class CPythonGeneralMappingTests(mapping_tests.BasicTestMappingProtocol): + + @classmethod + def setUpClass(cls): + cls.type2test = c_coll.OrderedDict + + def test_popitem(self): + d = self._empty_mapping() + self.assertRaises(KeyError, d.popitem) + + +class PurePythonSubclassMappingTests(mapping_tests.BasicTestMappingProtocol): + + @classmethod + def setUpClass(cls): + class MyOrderedDict(py_coll.OrderedDict): + pass + cls.type2test = MyOrderedDict + + def test_popitem(self): + d = self._empty_mapping() + self.assertRaises(KeyError, d.popitem) + + +@unittest.skipUnless(c_coll, 'requires the C version of the collections module') +class CPythonSubclassMappingTests(mapping_tests.BasicTestMappingProtocol): + + @classmethod + def setUpClass(cls): + class MyOrderedDict(c_coll.OrderedDict): + pass + cls.type2test = MyOrderedDict def test_popitem(self): d = self._empty_mapping() @@ -1391,7 +1502,10 @@ NamedTupleDocs = doctest.DocTestSuite(module=collections) test_classes = [TestNamedTuple, NamedTupleDocs, TestOneTrickPonyABCs, TestCollectionABCs, TestCounter, TestChainMap, - TestOrderedDict, GeneralMappingTests, SubclassMappingTests] + PurePythonOrderedDictTests, CPythonOrderedDictTests, + PurePythonGeneralMappingTests, CPythonGeneralMappingTests, + PurePythonSubclassMappingTests, CPythonSubclassMappingTests, + ] support.run_unittest(*test_classes) support.run_doctest(collections, verbose) diff -r 34986f55a32d Makefile.pre.in --- a/Makefile.pre.in Sun Oct 20 17:25:34 2013 +0300 +++ b/Makefile.pre.in Sun Oct 20 21:31:20 2013 +0300 @@ -412,6 +412,7 @@ Objects/listobject.o \ Objects/longobject.o \ Objects/dictobject.o \ + Objects/odictobject.o \ Objects/memoryobject.o \ Objects/methodobject.o \ Objects/moduleobject.o \ diff -r 34986f55a32d Modules/_collectionsmodule.c --- a/Modules/_collectionsmodule.c Sun Oct 20 17:25:34 2013 +0300 +++ b/Modules/_collectionsmodule.c Sun Oct 20 21:31:20 2013 +0300 @@ -1899,6 +1899,9 @@ Py_INCREF(&defdict_type); PyModule_AddObject(m, "defaultdict", (PyObject *)&defdict_type); + Py_INCREF(&PyODict_Type); + PyModule_AddObject(m, "OrderedDict", (PyObject *)&PyODict_Type); + if (PyType_Ready(&dequeiter_type) < 0) return NULL; Py_INCREF(&dequeiter_type); diff -r 34986f55a32d Objects/dict-common.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Objects/dict-common.h Sun Oct 20 21:31:20 2013 +0300 @@ -0,0 +1,22 @@ +#ifndef Py_DICT_COMMON_H +#define Py_DICT_COMMON_H + +typedef struct { + /* Cached hash code of me_key. */ + Py_hash_t me_hash; + PyObject *me_key; + PyObject *me_value; /* This field is only meaningful for combined tables */ +} PyDictKeyEntry; + +typedef PyDictKeyEntry *(*dict_lookup_func) +(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject ***value_addr); + +struct _dictkeysobject { + Py_ssize_t dk_refcnt; + Py_ssize_t dk_size; + dict_lookup_func dk_lookup; + Py_ssize_t dk_usable; + PyDictKeyEntry dk_entries[1]; +}; + +#endif diff -r 34986f55a32d Objects/dictobject.c --- a/Objects/dictobject.c Sun Oct 20 17:25:34 2013 +0300 +++ b/Objects/dictobject.c Sun Oct 20 21:31:20 2013 +0300 @@ -67,26 +67,9 @@ #define PyDict_MINSIZE_COMBINED 8 #include "Python.h" +#include "dict-common.h" #include "stringlib/eq.h" -typedef struct { - /* Cached hash code of me_key. */ - Py_hash_t me_hash; - PyObject *me_key; - PyObject *me_value; /* This field is only meaningful for combined tables */ -} PyDictKeyEntry; - -typedef PyDictKeyEntry *(*dict_lookup_func) -(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject ***value_addr); - -struct _dictkeysobject { - Py_ssize_t dk_refcnt; - Py_ssize_t dk_size; - dict_lookup_func dk_lookup; - Py_ssize_t dk_usable; - PyDictKeyEntry dk_entries[1]; -}; - /* To ensure the lookup algorithm terminates, there must be at least one Unused @@ -1362,6 +1345,141 @@ return 1; } +/* Internal version of dict.pop(). */ +PyObject * +_PyDict_Pop(PyDictObject *mp, PyObject *key, PyObject *deflt) +{ + Py_hash_t hash; + PyObject *old_value, *old_key; + PyDictKeyEntry *ep; + PyObject **value_addr; + + if (mp->ma_used == 0) { + if (deflt) { + Py_INCREF(deflt); + return deflt; + } + _PyErr_SetKeyError(key); + return NULL; + } + if (!PyUnicode_CheckExact(key) || + (hash = ((PyASCIIObject *) key)->hash) == -1) { + hash = PyObject_Hash(key); + if (hash == -1) + return NULL; + } + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); + if (ep == NULL) + return NULL; + old_value = *value_addr; + if (old_value == NULL) { + if (deflt) { + Py_INCREF(deflt); + return deflt; + } + _PyErr_SetKeyError(key); + return NULL; + } + *value_addr = NULL; + mp->ma_used--; + if (!_PyDict_HasSplitTable(mp)) { + ENSURE_ALLOWS_DELETIONS(mp); + old_key = ep->me_key; + Py_INCREF(dummy); + ep->me_key = dummy; + Py_DECREF(old_key); + } + return old_value; +} + +/* Internal version of dict.from_keys(). It is subclass-friendly. */ +PyObject * +_PyDict_FromKeys(PyObject *cls, PyObject *seq, PyObject *value) +{ + PyObject *it; /* iter(seq) */ + PyObject *key; + PyObject *d; + int status; + + d = PyObject_CallObject(cls, NULL); + if (d == NULL) + return NULL; + + if (PyDict_CheckExact(d) && ((PyDictObject *)d)->ma_used == 0) { + if (PyDict_CheckExact(seq)) { + PyDictObject *mp = (PyDictObject *)d; + PyObject *oldvalue; + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + + if (dictresize(mp, Py_SIZE(seq))) { + Py_DECREF(d); + return NULL; + } + + while (_PyDict_Next(seq, &pos, &key, &oldvalue, &hash)) { + if (insertdict(mp, key, hash, value)) { + Py_DECREF(d); + return NULL; + } + } + return d; + } + if (PyAnySet_CheckExact(seq)) { + PyDictObject *mp = (PyDictObject *)d; + Py_ssize_t pos = 0; + PyObject *key; + Py_hash_t hash; + + if (dictresize(mp, PySet_GET_SIZE(seq))) { + Py_DECREF(d); + return NULL; + } + + while (_PySet_NextEntry(seq, &pos, &key, &hash)) { + if (insertdict(mp, key, hash, value)) { + Py_DECREF(d); + return NULL; + } + } + return d; + } + } + + it = PyObject_GetIter(seq); + if (it == NULL){ + Py_DECREF(d); + return NULL; + } + + if (PyDict_CheckExact(d)) { + while ((key = PyIter_Next(it)) != NULL) { + status = PyDict_SetItem(d, key, value); + Py_DECREF(key); + if (status < 0) + goto Fail; + } + } else { + while ((key = PyIter_Next(it)) != NULL) { + status = PyObject_SetItem(d, key, value); + Py_DECREF(key); + if (status < 0) + goto Fail; + } + } + + if (PyErr_Occurred()) + goto Fail; + Py_DECREF(it); + return d; + +Fail: + Py_DECREF(it); + Py_DECREF(d); + return NULL; +} + /* Methods */ static void @@ -1692,91 +1810,9 @@ { PyObject *seq; PyObject *value = Py_None; - PyObject *it; /* iter(seq) */ - PyObject *key; - PyObject *d; - int status; - if (!PyArg_UnpackTuple(args, "fromkeys", 1, 2, &seq, &value)) return NULL; - - d = PyObject_CallObject(cls, NULL); - if (d == NULL) - return NULL; - - if (PyDict_CheckExact(d) && ((PyDictObject *)d)->ma_used == 0) { - if (PyDict_CheckExact(seq)) { - PyDictObject *mp = (PyDictObject *)d; - PyObject *oldvalue; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - - if (dictresize(mp, Py_SIZE(seq))) { - Py_DECREF(d); - return NULL; - } - - while (_PyDict_Next(seq, &pos, &key, &oldvalue, &hash)) { - if (insertdict(mp, key, hash, value)) { - Py_DECREF(d); - return NULL; - } - } - return d; - } - if (PyAnySet_CheckExact(seq)) { - PyDictObject *mp = (PyDictObject *)d; - Py_ssize_t pos = 0; - PyObject *key; - Py_hash_t hash; - - if (dictresize(mp, PySet_GET_SIZE(seq))) { - Py_DECREF(d); - return NULL; - } - - while (_PySet_NextEntry(seq, &pos, &key, &hash)) { - if (insertdict(mp, key, hash, value)) { - Py_DECREF(d); - return NULL; - } - } - return d; - } - } - - it = PyObject_GetIter(seq); - if (it == NULL){ - Py_DECREF(d); - return NULL; - } - - if (PyDict_CheckExact(d)) { - while ((key = PyIter_Next(it)) != NULL) { - status = PyDict_SetItem(d, key, value); - Py_DECREF(key); - if (status < 0) - goto Fail; - } - } else { - while ((key = PyIter_Next(it)) != NULL) { - status = PyObject_SetItem(d, key, value); - Py_DECREF(key); - if (status < 0) - goto Fail; - } - } - - if (PyErr_Occurred()) - goto Fail; - Py_DECREF(it); - return d; - -Fail: - Py_DECREF(it); - Py_DECREF(d); - return NULL; + return _PyDict_FromKeys(cls, seq, value); } static int @@ -2297,50 +2333,10 @@ static PyObject * dict_pop(PyDictObject *mp, PyObject *args) { - Py_hash_t hash; - PyObject *old_value, *old_key; PyObject *key, *deflt = NULL; - PyDictKeyEntry *ep; - PyObject **value_addr; - if(!PyArg_UnpackTuple(args, "pop", 1, 2, &key, &deflt)) return NULL; - if (mp->ma_used == 0) { - if (deflt) { - Py_INCREF(deflt); - return deflt; - } - _PyErr_SetKeyError(key); - return NULL; - } - if (!PyUnicode_CheckExact(key) || - (hash = ((PyASCIIObject *) key)->hash) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; - } - ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); - if (ep == NULL) - return NULL; - old_value = *value_addr; - if (old_value == NULL) { - if (deflt) { - Py_INCREF(deflt); - return deflt; - } - _PyErr_SetKeyError(key); - return NULL; - } - *value_addr = NULL; - mp->ma_used--; - if (!_PyDict_HasSplitTable(mp)) { - ENSURE_ALLOWS_DELETIONS(mp); - old_key = ep->me_key; - Py_INCREF(dummy); - ep->me_key = dummy; - Py_DECREF(old_key); - } - return old_value; + return _PyDict_Pop(mp, key, deflt); } static PyObject * @@ -2447,8 +2443,8 @@ static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); -static PyObject * -dict_sizeof(PyDictObject *mp) +PyObject * +_PyDict_SizeOf(PyDictObject *mp) { Py_ssize_t size, res; @@ -2520,7 +2516,7 @@ DICT___CONTAINS___METHODDEF {"__getitem__", (PyCFunction)dict_subscript, METH_O | METH_COEXIST, getitem__doc__}, - {"__sizeof__", (PyCFunction)dict_sizeof, METH_NOARGS, + {"__sizeof__", (PyCFunction)_PyDict_SizeOf, METH_NOARGS, sizeof__doc__}, {"get", (PyCFunction)dict_get, METH_VARARGS, get__doc__}, @@ -3041,8 +3037,8 @@ value = *value_ptr; Py_INCREF(key); Py_INCREF(value); - PyTuple_SET_ITEM(result, 0, key); - PyTuple_SET_ITEM(result, 1, value); + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ return result; fail: @@ -3137,28 +3133,22 @@ /* The instance lay-out is the same for all three; but the type differs. */ -typedef struct { - PyObject_HEAD - PyDictObject *dv_dict; -} dictviewobject; - - static void -dictview_dealloc(dictviewobject *dv) +dictview_dealloc(_PyDictViewObject *dv) { Py_XDECREF(dv->dv_dict); PyObject_GC_Del(dv); } static int -dictview_traverse(dictviewobject *dv, visitproc visit, void *arg) +dictview_traverse(_PyDictViewObject *dv, visitproc visit, void *arg) { Py_VISIT(dv->dv_dict); return 0; } static Py_ssize_t -dictview_len(dictviewobject *dv) +dictview_len(_PyDictViewObject *dv) { Py_ssize_t len = 0; if (dv->dv_dict != NULL) @@ -3166,10 +3156,10 @@ return len; } -static PyObject * -dictview_new(PyObject *dict, PyTypeObject *type) +PyObject * +_PyDictView_New(PyObject *dict, PyTypeObject *type) { - dictviewobject *dv; + _PyDictViewObject *dv; if (dict == NULL) { PyErr_BadInternalCall(); return NULL; @@ -3181,7 +3171,7 @@ type->tp_name, dict->ob_type->tp_name); return NULL; } - dv = PyObject_GC_New(dictviewobject, type); + dv = PyObject_GC_New(_PyDictViewObject, type); if (dv == NULL) return NULL; Py_INCREF(dict); @@ -3285,7 +3275,7 @@ } static PyObject * -dictview_repr(dictviewobject *dv) +dictview_repr(_PyDictViewObject *dv) { PyObject *seq; PyObject *result; @@ -3302,7 +3292,7 @@ /*** dict_keys ***/ static PyObject * -dictkeys_iter(dictviewobject *dv) +dictkeys_iter(_PyDictViewObject *dv) { if (dv->dv_dict == NULL) { Py_RETURN_NONE; @@ -3311,7 +3301,7 @@ } static int -dictkeys_contains(dictviewobject *dv, PyObject *obj) +dictkeys_contains(_PyDictViewObject *dv, PyObject *obj) { if (dv->dv_dict == NULL) return 0; @@ -3436,7 +3426,7 @@ PyObject *item = NULL; if (self == other) { - if (dictview_len((dictviewobject *)self) == 0) + if (dictview_len((_PyDictViewObject *)self) == 0) Py_RETURN_TRUE; else Py_RETURN_FALSE; @@ -3445,7 +3435,7 @@ /* Iterate over the shorter object (only if other is a set, * because PySequence_Contains may be expensive otherwise): */ if (PyAnySet_Check(other) || PyDictViewSet_Check(other)) { - Py_ssize_t len_self = dictview_len((dictviewobject *)self); + Py_ssize_t len_self = dictview_len((_PyDictViewObject *)self); Py_ssize_t len_other = PyObject_Size(other); if (len_other == -1) return NULL; @@ -3492,7 +3482,7 @@ PyTypeObject PyDictKeys_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_keys", /* tp_name */ - sizeof(dictviewobject), /* tp_basicsize */ + sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ (destructor)dictview_dealloc, /* tp_dealloc */ @@ -3525,13 +3515,13 @@ static PyObject * dictkeys_new(PyObject *dict) { - return dictview_new(dict, &PyDictKeys_Type); + return _PyDictView_New(dict, &PyDictKeys_Type); } /*** dict_items ***/ static PyObject * -dictitems_iter(dictviewobject *dv) +dictitems_iter(_PyDictViewObject *dv) { if (dv->dv_dict == NULL) { Py_RETURN_NONE; @@ -3540,7 +3530,7 @@ } static int -dictitems_contains(dictviewobject *dv, PyObject *obj) +dictitems_contains(_PyDictViewObject *dv, PyObject *obj) { PyObject *key, *value, *found; if (dv->dv_dict == NULL) @@ -3578,7 +3568,7 @@ PyTypeObject PyDictItems_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_items", /* tp_name */ - sizeof(dictviewobject), /* tp_basicsize */ + sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ (destructor)dictview_dealloc, /* tp_dealloc */ @@ -3611,13 +3601,13 @@ static PyObject * dictitems_new(PyObject *dict) { - return dictview_new(dict, &PyDictItems_Type); + return _PyDictView_New(dict, &PyDictItems_Type); } /*** dict_values ***/ static PyObject * -dictvalues_iter(dictviewobject *dv) +dictvalues_iter(_PyDictViewObject *dv) { if (dv->dv_dict == NULL) { Py_RETURN_NONE; @@ -3643,7 +3633,7 @@ PyTypeObject PyDictValues_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "dict_values", /* tp_name */ - sizeof(dictviewobject), /* tp_basicsize */ + sizeof(_PyDictViewObject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ (destructor)dictview_dealloc, /* tp_dealloc */ @@ -3676,7 +3666,7 @@ static PyObject * dictvalues_new(PyObject *dict) { - return dictview_new(dict, &PyDictValues_Type); + return _PyDictView_New(dict, &PyDictValues_Type); } /* Returns NULL if cannot allocate a new PyDictKeysObject, diff -r 34986f55a32d Objects/object.c --- a/Objects/object.c Sun Oct 20 17:25:34 2013 +0300 +++ b/Objects/object.c Sun Oct 20 21:31:20 2013 +0300 @@ -1717,6 +1717,30 @@ if (PyType_Ready(&PyDict_Type) < 0) Py_FatalError("Can't initialize dict type"); + if (PyType_Ready(&PyODict_Type) < 0) + Py_FatalError("Can't initialize OrderedDict type"); + + if (PyType_Ready(&PyODictKeys_Type) < 0) + Py_FatalError("Can't initialize odict_keys type"); + + if (PyType_Ready(&PyODictItems_Type) < 0) + Py_FatalError("Can't initialize odict_items type"); + + if (PyType_Ready(&PyODictValues_Type) < 0) + Py_FatalError("Can't initialize odict_values type"); + + if (PyType_Ready(&PyODictIterKey_Type) < 0) + Py_FatalError("Can't initialize odict_keyiterator type"); + + if (PyType_Ready(&PyODictIterItem_Type) < 0) + Py_FatalError("Can't initialize odict_itemiterator type"); + + if (PyType_Ready(&PyODictIterValue_Type) < 0) + Py_FatalError("Can't initialize odict_valueiterator type"); + + if (PyType_Ready(&PyODictIterReversed_Type) < 0) + Py_FatalError("Can't initialize odict_reversediterator type"); + if (PyType_Ready(&PySet_Type) < 0) Py_FatalError("Can't initialize set type"); diff -r 34986f55a32d Objects/odictobject.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Objects/odictobject.c Sun Oct 20 21:31:20 2013 +0300 @@ -0,0 +1,2492 @@ +/* Ordered Dictionary object implementation. + +This implementation is necessarily explicitly equivalent to the pure Python +OrderedDict class in Lib/collections/__init__.py. The strategy there +involves using a doubly-linked-list to capture the order. We keep to that +strategy, using a lower-level linked-list. + +About the Linked-List +===================== + +For the linked list we use a basic doubly-linked-list. Using a circularly- +linked-list does have some benefits, but they don't apply so much here +since OrderedDict is focused on the ends of the list (for the most part). +Furthermore, there are some features of generic linked-lists that we simply +don't need for OrderedDict. Thus a simple custom implementation meets our +needs. Alternatives to our simple approach include the QCIRCLE_* +macros from BSD's queue.h, and the linux's list.h. + +Getting O(1) Node Lookup +------------------------ + +One invariant of Python's OrderedDict is that it preserves time complexity +of dict's methods, particularly the O(1) operations. Simply adding a +linked-list on top of dict is not sufficient here; operations for nodes in +the middle of the linked-list implicitly require finding the node first. +With a simple linked-list like we're using, that is an O(n) operation. +Consequently, methods like __delitem__() would change from O(1) to O(n), +which is unacceptable. + +In order to preserve O(1) performance for node removal (finding nodes), we +must do better than just looping through the linked-list. Here are options +we've considered: + +1. use a second dict to map keys to nodes (a la the pure Python version). +2. keep a simple hash table mirroring the order of dict's, mapping each key + to the corresponding node in the linked-list. +3. use a version of shared keys (split dict) that allows non-unicode keys. +4. have the value stored for each key be a (value, node) pair, and adjust + __getitem__(), get(), etc. accordingly. + +The approach with the least performance impact (time and space) is #2, +mirroring the key order of dict's dk_enties with an array of node pointers. +While lookdict() and friends (dk_lookup) don't give us the index into the +array, we make use of pointer arithmetic to get that index. An alternative +would be to refactor lookdict() to provide the index, explicitly exposing +the implementation detail. We could even just use a custom lookup function +for OrderedDict that facilitates our need. However, both approaches are +significantly more complicated than just using pointer arithmetic. + +The catch with mirroring the hash table ordering is that we have to keep +the ordering in sync through any dict resizes. However, that order only +matters during node lookup. We can simply defer any potential resizing +until we need to do a lookup. + +Linked-List Nodes +----------------- + +The current implementation stores a pointer to the associated key only. +One alternative would be to store a pointer to the PyDictKeyEntry instead. +This would save one pointer de-reference per item, which is nice during +calls to values() and items(). However, it adds unnecessary overhead +otherwise, so we stick with just the key. + +Linked-List API +--------------- + +As noted, the linked-list implemented here does not have all the bells and +whistles. However, we recognize that the implementation may need to +change to accommodate performance improvements or extra functionality. To +that end, We use a simple API to interact with the linked-list. Here's a +summary of the methods/macros: + +Node info: + +* _odictnode_KEY(node) +* _odictnode_VALUE(od, node) +* _odictnode_PREV(node) +* _odictnode_NEXT(node) + +Linked-List info: + +* _odict_FIRST(od) +* _odict_LAST(od) +* _odict_EMPTY(od) +* _odict_FOREACH(od, node) - used in place of `for (node=...)` + +For adding nodes: + +* _odict_add_head(od, node) +* _odict_add_tail(od, node) +* _odict_add_new_node(od, key) + +For removing nodes: + +* _odict_pop_node(od, node, key) +* _odict_clear_node(od, node) +* _odict_clear_nodes(od, clear_each) + +Others: + +* _odict_initialize(od) +* _odict_find_node(od, key) +* _odict_keys_equal(od1, od2) + +Used, but specific to the linked-list implementation: + +* _odict_free_fast_nodes(od) + +And here's a look at how the linked-list relates to the OrderedDict API: + +============ === === ==== ==== ==== === ==== ===== ==== ==== === ==== === === +method key val prev next mem 1st last empty iter find add rmv clr keq +============ === === ==== ==== ==== === ==== ===== ==== ==== === ==== === === +__del__ ~ X +__delitem__ free ~ node +__eq__ ~ X +__iter__ X X +__new__ X X +__reduce__ X ~ X +__repr__ X X X +__reversed__ X X +__setitem__ key +__sizeof__ size X +clear ~ ~ X +copy X X X +items X X X +keys X X +move_to_end X X X ~ h/t key +pop free key +popitem X X free X X node +setdefault ~ ? ~ +values X X +============ === === ==== ==== ==== === ==== ===== ==== ==== === ==== === === + +__delitem__ is the only method that directly relies on finding an arbitrary +node in the linked-list. Everything else is iteration or relates to the +ends of the lisked-list. + +Situation that Endangers Consistency +------------------------------------ +Using a raw linked-list for OrderedDict exposes a key situation that can +cause problems. If a node is stored in a variable, there is a chance that +the node may have been deallocated before the variable gets used, thus +potentially leading to a segmentation fault. A key place where this shows +up is during iteration through the linked list (via _odict_FOREACH or +otherwise). + +A number of solutions are available to resolve this situation: + +* defer looking up the node until as late as possible and certainly after + any code that could possibly result in a deletion; +* if the node is needed both before and after a point where the node might + be removed, do a check before using the node at the "after" location to + see if the node is still valid; +* like the last one, but simply pull the node again to ensure it's right; +* keep the key in the variable instead of the node and then look up the + node using the key at the point where the node is needed (this is what + we do for the iterators). + +XXX What if the node is removed and another is allocated to the same +address? What if it even has the same key? + +Another related problem, preserving consistent ordering during iteration, +is described below. That one is not exclusive to using linked-lists. + +Challenges from Subclassing dict +================================ + +OrderedDict subclasses dict, which is an unusual relationship between two +builtin types (other than the base object type). Doing so results in +some complication and deserves further explanation. There are two things +to consider here. First, in what circumstances or with what adjustments +can OrderedDict be used as a drop-in replacement for dict (at the C level)? +Second, how can the OrderedDict implementation leverage the dict +implementation effectively without introducing unnecessary coupling or +inefficiencies? + +This second point is reflected here and in the implementation, so the +further focus is on the first point. It is worth noting that for +overridden methods, the dict implementation is deferred to as much as +possible. Furthermore, coupling is limited to as little as is reasonable. + +Concrete API Compatibility +-------------------------- + +Use of the concrete C-API for dict (PyDict_*) with OrderedDict is +problematic. (See http://bugs.python.org/issue10977.) The concrete API +has a number of hard-coded assumptions tied to the dict implementation. +This is, in part, due to performance reasons, which is understandable +given the part dict plays in Python. + +Any attempt to replace dict with OrderedDict for any role in the +interpreter (e.g. **kwds) faces a challenge. Such any effort must +recognize that the instances in affected locations currently interact with +the concrete API. + +Here are some ways to address this challenge: + +1. Change the relevant usage of the concrete API in CPython and add + PyDict_CheckExact() calls to each of the concrete API funcions. +2. Adjust the relevant concrete API functions to explicitly accommodate + OrderedDict. +3. As with #1, add the checks, but improve the abstract API with smart fast + paths for dict and OrderedDict, and refactor CPython to use the abstract + API. Improvements to the abstract API would be valuable regardless. + +Adding the checks to the concrete API would help make any interpreter +switch to OrderedDict less painful for extension modules. However, this +won't work. The equivalent C API call to `dict.__setitem__(obj, k, v)` +is 'PyDict_SetItem(obj, k, v)`. This illustrates how subclasses in C call +the base class's methods, since there is no equivalent of super() in the +C API. Calling into Python for parent class API would work, but some +extension modules already rely on this feature of the concrete API. + +For reference, here is a breakdown of some of the dict concrete API: + +========================== ============= ======================= +concrete API uses abstract API +========================== ============= ======================= +PyDict_Check PyMapping_Check +(PyDict_CheckExact) - +(PyDict_New) - +(PyDictProxy_New) - +PyDict_Clear - +PyDict_Contains PySequence_Contains +PyDict_Copy - +PyDict_SetItem PyObject_SetItem +PyDict_SetItemString PyMapping_SetItemString +PyDict_DelItem PyMapping_DelItem +PyDict_DelItemString PyMapping_DelItemString +PyDict_GetItem - +PyDict_GetItemWithError PyObject_GetItem +_PyDict_GetItemIdWithError - +PyDict_GetItemString PyMapping_GetItemString +PyDict_Items PyMapping_Items +PyDict_Keys PyMapping_Keys +PyDict_Values PyMapping_Values +PyDict_Size PyMapping_Size + PyMapping_Length +PyDict_Next PyIter_Next +_PyDict_Next - +PyDict_Merge - +PyDict_Update - +PyDict_MergeFromSeq2 - +PyDict_ClearFreeList - +- PyMapping_HasKeyString +- PyMapping_HasKey +========================== ============= ======================= + + +The dict Interface Relative to OrderedDict +========================================== + +Since OrderedDict subclasses dict, understanding the various methods and +attributes of dict is important for implementing OrderedDict. + +Relevant Type Slots +------------------- + +================= ================ =================== ================ +slot attribute object dict +================= ================ =================== ================ +tp_dealloc - object_dealloc dict_dealloc +tp_repr __repr__ object_repr dict_repr +sq_contains __contains__ - dict_contains +mp_length __len__ - dict_length +mp_subscript __getitem__ - dict_subscript +mp_ass_subscript __setitem__ - dict_ass_sub + __delitem__ +tp_hash __hash__ _Py_HashPointer ..._HashNotImpl +tp_str __str__ object_str - +tp_getattro __getattribute__ ..._GenericGetAttr (repeated) + __getattr__ +tp_setattro __setattr__ ..._GenericSetAttr (disabled) +tp_doc __doc__ (literal) dictionary_doc +tp_traverse - - dict_traverse +tp_clear - - dict_tp_clear +tp_richcompare __eq__ object_richcompare dict_richcompare + __ne__ +tp_weaklistoffset (__weakref__) - - +tp_iter __iter__ - dict_iter +tp_dictoffset (__dict__) - - +tp_init __init__ object_init dict_init +tp_alloc - PyType_GenericAlloc (repeated) +tp_new __new__ object_new dict_new +tp_free - PyObject_Del PyObject_GC_Del +================= ================ =================== ================ + +Relevant Methods +---------------- + +================ =================== =============== +method object dict +================ =================== =============== +__reduce__ object_reduce - +__sizeof__ object_sizeof dict_sizeof +clear - dict_clear +copy - dict_copy +fromkeys - dict_fromkeys +get - dict_get +items - dictitems_new +keys - dictkeys_new +pop - dict_pop +popitem - dict_popitem +setdefault - dict_setdefault +update - dict_update +values - dictvalues_new +================ =================== =============== + + +Pure Python OrderedDict +======================= + +As already noted, compatibility with the pure Python OrderedDict +implementation is a key goal of this C implementation. To further that +goal, here's a summary of how OrderedDict-specific methods are implemented +in collections/__init__.py. Also provided is an indication of which +methods directly mutate or iterate the object, as well as any relationship +with the underlying linked-list. + +============= ============== == ================ === === ==== +method impl used ll uses inq mut iter +============= ============== == ================ === === ==== +__contains__ dict - - X +__delitem__ OrderedDict Y dict.__delitem__ X +__eq__ OrderedDict N OrderedDict ~ + dict.__eq__ + __iter__ +__getitem__ dict - - X +__iter__ OrderedDict Y - X +__init__ OrderedDict N update +__len__ dict - - X +__ne__ MutableMapping - __eq__ ~ +__reduce__ OrderedDict N OrderedDict ~ + __iter__ + __getitem__ +__repr__ OrderedDict N __class__ ~ + items +__reversed__ OrderedDict Y - X +__setitem__ OrderedDict Y __contains__ X + dict.__setitem__ +__sizeof__ OrderedDict Y __len__ ~ + __dict__ +clear OrderedDict Y dict.clear X +copy OrderedDict N __class__ + __init__ +fromkeys OrderedDict N __setitem__ +get dict - - ~ +items MutableMapping - ItemsView X +keys MutableMapping - KeysView X +move_to_end OrderedDict Y - X +pop OrderedDict N __contains__ X + __getitem__ + __delitem__ +popitem OrderedDict Y dict.pop X +setdefault OrderedDict N __contains__ ~ + __getitem__ + __setitem__ +update MutableMapping - __setitem__ ~ +values MutableMapping - ValuesView X +============= ============== == ================ === === ==== + +__reversed__ and move_to_end are both exclusive to OrderedDict. + + +C OrderedDict Implementation +============================ + +================= ================ +slot or method impl +================= ================ +tp_dealloc odict_dealloc +tp_repr odict_repr +mp_ass_subscript odict_ass_sub +tp_doc odict_doc +tp_traverse odict_traverse +tp_clear odict_tp_clear +tp_richcompare odict_richcompare +tp_weaklistoffset (offset) +tp_iter odict_iter +tp_dictoffset (offset) +tp_init odict_init +tp_alloc (repeated) +tp_new odict_new + +__reduce__ odict_reduce +__sizeof__ odict_sizeof +clear odict_clear +copy odict_copy +fromkeys odict_fromkeys +items odictitems_new +keys odictkeys_new +pop odict_pop +popitem odict_popitem +setdefault odict_setdefault +update odict_update +values odictvalues_new +================= ================ + +XXX Can't we just use dict's fromkeys? + +Inherited unchanged from object/dict: + +================ ========================== +method type field +================ ========================== +- tp_free +__contains__ tp_as_sequence.sq_contains +__getattr__ tp_getattro +__getattribute__ tp_getattro +__getitem__ tp_as_mapping.mp_subscript +__hash__ tp_hash +__len__ tp_as_mapping.mp_length +__setattr__ tp_setattro +__str__ tp_str +get - +================ ========================== + + +Other Challenges +================ + +Preserving Ordering During Iteration +------------------------------------ +During iteration through an OrderedDict, it is possible that items could +get added, removed, or reordered. For a linked-list implementation, as +with some other implementations, that situation may lead to undefined +behavior. The documentation for dict mentions this in the `iter()` section +of http://docs.python.org/3.4/library/stdtypes.html#dictionary-view-objects. +In this implementation we follow dict's lead (as does the pure Python +implementation) for __iter__(), keys(), values(), and items(). + +For internal iteration (using _odict_FOREACH or not), there is still the +risk that not all nodes that we expect to be seen in the loop actually get +seen. Thus, we are careful in each of those places to ensure that they +are. This comes, of course, at a small price at each location. The +solutions are much the same as those detailed in the `Situation that +Endangers Consistency` section above. + + +Potential Optimizations +======================= + +* Allocate the nodes as a block via od_fast_nodes instead of individually. + - Set node->key to NULL to indicate the node is not-in-use. + - Add _odict_EXISTS()? + - How to maintain consistency across resizes? Existing node pointers + would be invalidate after a resize, which is particularly problematic + for the iterators. +* Use a more stream-lined implementation of update() and, likely indirectly, + __init__(). + +*/ + +/* TODO + +sooner: +- reentrancy (make sure everything is at a thread-safe state when calling + into Python) + +later: +- make the dict views support the full set API (the pure Python impl does) +- implement a fuller MutableMapping API in C? + +*/ + +#include "Python.h" +#include "structmember.h" +#include "dict-common.h" +#include + + +typedef struct _odictnode _ODictNode; + +/* PyODictObject */ +struct _odictobject { + PyDictObject od_dict; + _ODictNode *od_first; + _ODictNode *od_last; + Py_ssize_t od_size; /* managed by _odict_resize() */ + _ODictNode **od_fast_nodes; /* managed by _odict_resize() */ + PyObject *od_inst_dict; + PyObject *od_weakreflist; +}; + + +/* ---------------------------------------------- + * odict keys (a simple doubly-linked list) + */ + +struct _odictnode { + PyObject *key; + _ODictNode *next; + _ODictNode *prev; +}; + +#define _odictnode_KEY(node) \ + (node->key) +/* borrowed reference */ +/* XXX use PyDict_GetItemWithError()? */ +#define _odictnode_VALUE(node, od) \ + PyODict_GetItem((PyObject *)od, _odictnode_KEY(node)) +/* If needed we could also have _odictnode_HASH. */ +#define _odictnode_PREV(node) (node->prev) +#define _odictnode_NEXT(node) (node->next) + +#define _odict_FIRST(od) (((PyODictObject *)od)->od_first) +#define _odict_LAST(od) (((PyODictObject *)od)->od_last) +#define _odict_EMPTY(od) (_odict_FIRST(od) == NULL) +#define _odict_FOREACH(od, node) \ + for (node = _odict_FIRST(od); node != NULL; node = _odictnode_NEXT(node)) + + +static Py_ssize_t _odict_get_index(PyODictObject *, PyObject *); /* forward */ + +static void +_odict_free_fast_nodes(PyODictObject *od) { + if (od->od_fast_nodes) { + PyMem_FREE(od->od_fast_nodes); + } +} + +static int +_odict_resize(PyODictObject *od) { + Py_ssize_t size, prev_size, i; + _ODictNode **fast_nodes, *node; + +start: + size = ((PyDictObject *)od)->ma_keys->dk_size; + fast_nodes = PyMem_NEW(_ODictNode *, size); + if (fast_nodes == NULL) { + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < size; i++) + fast_nodes[i] = NULL; + + prev_size = od->od_size; + od->od_size = size; + _odict_FOREACH(od, node) { + size_t i = _odict_get_index(od, _odictnode_KEY(node)); + if (i < 0) { + od->od_size = prev_size; + PyMem_FREE(fast_nodes); + return -1; + } + fast_nodes[i] = node; + } + if (size != ((PyDictObject *)od)->ma_keys->dk_size) { + /* Must have resized during the _odict_get_index() call. */ + PyMem_FREE(fast_nodes); + goto start; + } + + _odict_free_fast_nodes(od); + od->od_fast_nodes = fast_nodes; + return 0; +} + +/* Return the index into the hash table, regardless of a valid node. */ +static Py_ssize_t +_odict_get_index(PyODictObject *od, PyObject *key) +{ + Py_hash_t hash; + PyObject **value_addr = NULL; + PyDictKeyEntry *ep; + PyDictKeysObject *keys = ((PyDictObject *)od)->ma_keys; + +start: + /* Ensure od_fast_nodes and dk_entries are in sync. */ + if (keys->dk_size != od->od_size) { + int resize_res = _odict_resize(od); + if (resize_res < 0) + return -1; + } + + /* now get the index */ + hash = PyObject_Hash(key); + if (keys->dk_size != od->od_size) { + /* Must have resized during the PyObject_Hash() call. */ + goto start; + } + + ep = (keys->dk_lookup)((PyDictObject *)od, key, hash, &value_addr); + if (ep == NULL) + return -1; + return ep - keys->dk_entries; +} + +static int +_odict_initialize(PyODictObject *od) +{ + _odict_FIRST(od) = NULL; + _odict_LAST(od) = NULL; + return _odict_resize((PyODictObject *)od); +} + +/* Returns NULL if there was some error or the key was not found. */ +static _ODictNode * +_odict_find_node(PyODictObject *od, PyObject *key) +{ + Py_ssize_t index; + + if (_odict_EMPTY(od)) + return NULL; + index = _odict_get_index(od, key); + if (index < 0) + return NULL; + return od->od_fast_nodes[index]; +} + +static void +_odict_add_head(PyODictObject *od, _ODictNode *node) +{ + _odictnode_PREV(node) = NULL; + if (_odict_FIRST(od) == NULL) { + _odict_LAST(od) = node; + } + else { + _odictnode_PREV(_odict_FIRST(od)) = node; + } + _odictnode_NEXT(node) = _odict_FIRST(od); + _odict_FIRST(od) = node; +} + +static void +_odict_add_tail(PyODictObject *od, _ODictNode *node) +{ + _odictnode_NEXT(node) = NULL; + if (_odict_LAST(od) == NULL) { + _odict_FIRST(od) = node; + } + else { + _odictnode_NEXT(_odict_LAST(od)) = node; + } + _odictnode_PREV(node) = _odict_LAST(od); + _odict_LAST(od) = node; +} + +/* adds the node to the end of the list */ +static int +_odict_add_new_node(PyODictObject *od, PyObject *key) +{ + Py_ssize_t i; + _ODictNode *node; + + Py_INCREF(key); + i = _odict_get_index(od, key); + if (i < 0) { + Py_DECREF(key); + if (!PyErr_Occurred()) + PyErr_SetObject(PyExc_KeyError, key); + return -1; + } + else if (od->od_fast_nodes[i] != NULL) { + /* Already added! */ + Py_DECREF(key); + return 0; + } + + /* must not be added yet */ + node = (_ODictNode *)PyMem_MALLOC(sizeof(_ODictNode)); + if (node == NULL) { + Py_DECREF(key); + PyErr_NoMemory(); + return -1; + } + + _odictnode_KEY(node) = key; + _odict_add_tail(od, node); + od->od_fast_nodes[i] = node; + return 0; +} + +/* Putting the decref after the free causes problems. */ +#define _odictnode_DEALLOC(node) \ + do { \ + Py_DECREF(_odictnode_KEY(node)); \ + PyMem_FREE((void *)node); \ + } while (0) + +/* Repeated calls on the same node are no-ops. */ +static void +_odict_remove_node(PyODictObject *od, _ODictNode *node) +{ + if (_odict_FIRST(od) == node) + _odict_FIRST(od) = _odictnode_NEXT(node); + else if (_odictnode_PREV(node) != NULL) + _odictnode_NEXT(_odictnode_PREV(node)) = _odictnode_NEXT(node); + + if (_odict_LAST(od) == node) + _odict_LAST(od) = _odictnode_PREV(node); + else if (_odictnode_NEXT(node) != NULL) + _odictnode_PREV(_odictnode_NEXT(node)) = _odictnode_PREV(node); + + _odictnode_PREV(node) = NULL; + _odictnode_NEXT(node) = NULL; +} + +static _ODictNode * +_odict_pop_node(PyODictObject *od, _ODictNode *node, PyObject *key) +{ + if (node == NULL) { + node = _odict_find_node(od, key); + if (node == NULL) + return NULL; + } + _odict_remove_node(od, node); + return node; +} + +/* If someone calls PyDict_DelItem() directly on an OrderedDict, we'll + get all sorts of problems here. +*/ +static int +_odict_clear_node(PyODictObject *od, _ODictNode *node, PyObject *key) +{ + Py_ssize_t i; + + if (_odict_EMPTY(od)) + return 0; + + if (node == NULL) { + i = _odict_get_index(od, key); + if (i >= 0) + node = od->od_fast_nodes[i]; + } + else { + if (key == NULL) + key = _odictnode_KEY(node); + i = _odict_get_index(od, key); + } + if (i < 0) + return PyErr_Occurred() ? -1 : 0; + + /* Remove the node if it hasn't been already. */ + if (od->od_fast_nodes[i] != NULL) { + od->od_fast_nodes[i] = NULL; + _odict_remove_node(od, node); + _odictnode_DEALLOC(node); + } + return 0; +} + +static void +_odict_clear_nodes(PyODictObject *od, int clear_each) +{ + _ODictNode *node, *next; + + if (_odict_EMPTY(od)) + return; + + node = _odict_FIRST(od); + while (node != NULL) { + next = _odictnode_NEXT(node); + if (clear_each) { + _odict_clear_node(od, node, NULL); + /* XXX node may be out of date due to _odict_clear_node(). */ + } + else { + /* The caller will remove the node from the linked-list + separately or just doesn't care. */ + _odictnode_DEALLOC(node); + } + node = next; + } + _odict_FIRST(od) = NULL; + _odict_LAST(od) = NULL; + + _odict_free_fast_nodes(od); + od->od_fast_nodes = NULL; +} + +#undef _odictnode_DEALLOC + +static int +_odict_keys_equal(PyODictObject *a, PyODictObject *b) +{ + _ODictNode *node_a, *node_b; + + node_a = _odict_FIRST(a); + node_b = _odict_FIRST(b); + while (1) { + if (node_a == NULL && node_b == NULL) + /* success: hit the end of each at the same time */ + return 1; + else if (node_a == NULL || node_b == NULL) + /* unequal length */ + return 0; + else { + int res = PyObject_RichCompareBool( + (PyObject *)_odictnode_KEY(node_a), + (PyObject *)_odictnode_KEY(node_b), + Py_EQ); + if (res < 0) + return res; + else if (res == 0) + return 0; + + /* XXX node_a and/or node_b may be out of date if either + * odict mutated during the PyObject_RichCompareBool() call. + */ + + /* otherwise it must match, so move on to the next one */ + node_a = _odictnode_NEXT(node_a); + node_b = _odictnode_NEXT(node_b); + } + } +} + + +/* ---------------------------------------------- + * OrderedDict mapping methods + */ + +/* mp_ass_subscript: __setitem__() and __delitem__() */ + +static int +odict_mp_ass_sub(PyODictObject *od, PyObject *v, PyObject *w) +{ + if (w == NULL) + return PyODict_DelItem((PyObject *)od, v); + else + return PyODict_SetItem((PyObject *)od, v, w); +} + +/* tp_as_mapping */ + +static PyMappingMethods odict_as_mapping = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + (objobjargproc)odict_mp_ass_sub, /*mp_ass_subscript*/ +}; + + +/* ---------------------------------------------- + * OrderedDict methods + */ + +/* __delitem__() */ + +PyDoc_STRVAR(odict_delitem__doc__, "od.__delitem__(y) <==> del od[y]"); + +/* __eq__() */ + +PyDoc_STRVAR(odict_eq__doc__, +"od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive \n\ + while comparison to a regular mapping is order-insensitive.\n\ + "); + +/* forward */ +static PyObject * odict_richcompare(PyObject *v, PyObject *w, int op); + +static PyObject * +odict_eq(PyObject *a, PyObject *b) +{ + return odict_richcompare(a, b, Py_EQ); +} + +/* __init__() */ + +PyDoc_STRVAR(odict_init__doc__, +"Initialize an ordered dictionary. The signature is the same as\n\ + regular dictionaries, but keyword arguments are not recommended because\n\ + their insertion order is arbitrary.\n\ +\n\ + "); + +/* forward */ +static int odict_init(PyObject *self, PyObject *args, PyObject *kwds); + +/* __iter__() */ + +PyDoc_STRVAR(odict_iter__doc__, "od.__iter__() <==> iter(od)"); + +static PyObject * odict_iter(PyODictObject *self); /* forward */ + +/* __ne__() */ + +/* Mapping.__ne__() does not have a docstring. */ +PyDoc_STRVAR(odict_ne__doc__, ""); + +static PyObject * +odict_ne(PyObject *a, PyObject *b) +{ + return odict_richcompare(a, b, Py_NE); +} + +/* __repr__() */ + +PyDoc_STRVAR(odict_repr__doc__, "od.__repr__() <==> repr(od)"); + +static PyObject * odict_repr(PyODictObject *self); /* forward */ + +/* __setitem__() */ + +PyDoc_STRVAR(odict_setitem__doc__, "od.__setitem__(i, y) <==> od[i]=y"); + +/* fromkeys() */ + +PyDoc_STRVAR(odict_fromkeys__doc__, +"OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.\n\ + If not specified, the value defaults to None.\n\ +\n\ + "); + +static PyObject * +odict_fromkeys(PyObject *cls, PyObject *args) +{ + PyObject *seq; + PyObject *value = Py_None; + if (!PyArg_UnpackTuple(args, "fromkeys", 1, 2, &seq, &value)) /* borrowed */ + return NULL; + return _PyDict_FromKeys(cls, seq, value); +} + +/* __sizeof__() */ + +/* OrderedDict.__sizeof__() does not have a docstring. */ +PyDoc_STRVAR(odict_sizeof__doc__, ""); + +static PyObject * +odict_sizeof(PyODictObject *od) +{ + PyObject *pylong; + Py_ssize_t res; + + /* XXX May mutate during the Py* calls. */ + + pylong = _PyDict_SizeOf((PyDictObject *)od); + if (pylong == NULL) + return NULL; + res = PyLong_AsSsize_t(pylong); + Py_DECREF(pylong); + if (res == -1 && PyErr_Occurred()) + return NULL; + + res += sizeof(PyODictObject) - sizeof(PyDictObject); + + /* instance dict */ + pylong = _PyDict_SizeOf((PyDictObject *)od->od_inst_dict); + if (pylong == NULL) + return NULL; + res += PyLong_AsSsize_t(pylong); + Py_DECREF(pylong); + if (res == -1 && PyErr_Occurred()) + return NULL; + + /* XXX weakrefs? (od->od_weakreflist) */ + + res += sizeof(_ODictNode) * od->od_size; /* od_fast_nodes */ + if (!_odict_EMPTY(od)) { + res += sizeof(_ODictNode) * PyODict_SIZE(od); /* linked-list */ + } + return PyLong_FromSsize_t(res); +} + +/* __reduce__() */ + +PyDoc_STRVAR(odict_reduce__doc__, "Return state information for pickling"); + +static PyObject * +odict_reduce(register PyODictObject *od) +{ + _Py_IDENTIFIER(__dict__); + _Py_IDENTIFIER(__class__); + PyObject *vars = NULL, *ns = NULL, *result = NULL, *cls = NULL; + PyObject *items_iter = NULL, *items = NULL, *args = NULL; + + /* capture any instance state */ + vars = _PyObject_GetAttrId((PyObject *)od, &PyId___dict__); + if (vars != NULL) { + PyObject *empty, *od_vars, *iterator, *key; + + /* od.__dict__ isn't necessarily a dict... */ + ns = PyObject_CallMethod((PyObject *)vars, "copy", NULL); + if (ns == NULL) + goto Done; + empty = PyODict_New(); + if (empty == NULL) + goto Done; + od_vars = _PyObject_GetAttrId((PyObject *)empty, &PyId___dict__); + Py_DECREF(empty); + if (od_vars == NULL) + goto Done; + iterator = PyObject_GetIter(od_vars); + Py_DECREF(od_vars); + if (iterator == NULL) + goto Done; + + while ( (key = PyIter_Next(iterator)) ) { + /* XXX set an exception for when a key is in empty but not in ns? */ + if (PyMapping_HasKey(ns, key) && PyMapping_DelItem(ns, key) != 0) { + Py_DECREF(iterator); + Py_DECREF(key); + goto Done; + } + Py_DECREF(key); + } + Py_DECREF(iterator); + if (PyErr_Occurred()) + goto Done; + + if (!PyObject_Length(ns)) { + /* nothing novel to pickle in od.__dict__ */ + Py_DECREF(ns); + ns = NULL; + } + } + + /* build the result */ + cls = _PyObject_GetAttrId((PyObject *)od, &PyId___class__); + if (cls == NULL) + goto Done; + + args = PyTuple_New(0); + if (args == NULL) + goto Done; + + items = PyObject_CallMethod((PyObject *)od, "items", NULL); + if (items == NULL) + goto Done; + + items_iter = PyObject_GetIter(items); + if (items_iter == NULL) + goto Done; + + result = PyTuple_Pack(5, cls, args, ns ? ns : Py_None, Py_None, items_iter); + +Done: + Py_XDECREF(vars); + Py_XDECREF(ns); + Py_XDECREF(cls); + Py_XDECREF(args); + Py_XDECREF(items); + Py_XDECREF(items_iter); + + return result; +} + +/* setdefault() */ + +PyDoc_STRVAR(odict_setdefault__doc__, + "od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od"); + +/* Skips __missing__() calls. */ +static PyObject * +odict_setdefault(register PyODictObject *od, PyObject *args) +{ + _ODictNode *node; + PyObject *key, *result = NULL; + PyObject *failobj = Py_None; + + /* both borrowed */ + if (!PyArg_UnpackTuple(args, "setdefault", 1, 2, &key, &failobj)) + return NULL; + Py_INCREF(key); + Py_INCREF(failobj); + + if (PyODict_CheckExact(od)) { + node = _odict_find_node(od, key); + if (node == NULL) { + if (PyErr_Occurred()) { + goto done; + } + else if (PyODict_SetItem((PyObject *)od, key, failobj) >= 0) { + result = failobj; + Py_INCREF(failobj); + } + } + else { + result = PyODict_GetItem(od, key); /* borrowed reference */ + Py_XINCREF(result); + } + } + else { + int exists = PySequence_Contains((PyObject *)od, key); + if (exists < 0) { + goto done; + } + else if (exists) { + result = PyObject_GetItem((PyObject *)od, key); + } + else if (PyObject_SetItem((PyObject *)od, key, failobj) >= 0) { + result = failobj; + Py_INCREF(failobj); + } + } + +done: + Py_DECREF(failobj); + Py_DECREF(key); + return result; +} + +/* pop() */ + +PyDoc_STRVAR(odict_pop__doc__, +"od.pop(k[,d]) -> v, remove specified key and return the corresponding\n\ + value. If key is not found, d is returned if given, otherwise KeyError\n\ + is raised.\n\ +\n\ + "); + +/* Skips __missing__() calls. */ +static PyObject * +odict_pop(PyObject *od, PyObject *args) +{ + _ODictNode *node; + PyObject *key, *failobj = NULL, *value = NULL; + + if (!PyArg_UnpackTuple(args, "pop", 1, 2, &key, &failobj)) { /* borrowed */ + return NULL; + } + Py_INCREF(key); + Py_XINCREF(failobj); + + if (PyODict_CheckExact(od)) { + node = _odict_find_node((PyODictObject *)od, key); + if (node == NULL) { + if (PyErr_Occurred()) + goto done; + } + else { + /* We could do PyDict_GetItem() and PyDict_DelItem() directly... */ + value = _PyDict_Pop((PyDictObject *)od, key, failobj); + } + } + else { + int exists = PySequence_Contains(od, key); + if (exists < 0) + goto done; + if (exists) { + value = PyObject_GetItem(od, key); + if (value != NULL) { + if (PyObject_DelItem(od, key) == -1) { + Py_DECREF(value); + value = NULL; + } + } + } + } + + if (value != NULL) { + /* Make sure we are using the up-to-date node. */ + node = _odict_find_node((PyODictObject *)od, key); + if (node == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(value); + value = NULL; + goto done; + } + } + else { + int res = _odict_clear_node((PyODictObject *)od, node, key); + if (res < 0) { + Py_DECREF(value); + value = NULL; + } + } + } + else if (!PyErr_Occurred()) { + if (failobj) { + value = failobj; + Py_INCREF(failobj); + } + else { + PyErr_SetObject(PyExc_KeyError, key); + } + } + +done: + Py_DECREF(key); + Py_XDECREF(failobj); + return value; +} + +/* popitem() */ + +PyDoc_STRVAR(odict_popitem__doc__, +"od.popitem() -> (k, v), return and remove a (key, value) pair.\n\ + Pairs are returned in LIFO order if last is true or FIFO order if false.\n\ +\n\ + "); + +static PyObject * +odict_popitem(PyObject *od, PyObject *args) +{ + PyObject *key, *value, *item = NULL, *last = NULL; + _ODictNode *node; + int res; + + if (_odict_EMPTY((PyODictObject *)od)) { + PyErr_SetString(PyExc_KeyError, "dictionary is empty"); + return NULL; + } + + /* pull the item */ + + if (!PyArg_UnpackTuple(args, "popitem", 0, 1, &last)) /* borrowed */ + return NULL; + Py_XINCREF(last); + + if (last == NULL || last == Py_True) + node = _odict_LAST((PyODictObject *)od); + else + node = _odict_FIRST((PyODictObject *)od); + Py_XDECREF(last); + + key = _odictnode_KEY(node); + if (PyODict_CheckExact(od)) { + value = _odictnode_VALUE(node, (PyODictObject *)od); /* borrowed */ + Py_XINCREF(value); + } + else { + value = PyObject_GetItem(od, key); + } + if (value == NULL) + return NULL; + + /* XXX Could be a problem as described in dict_popitem()? */ + item = PyTuple_Pack(2, key, value); + Py_DECREF(value); + if (item == NULL) + return NULL; + + /* remove it from the mapping */ + if (PyODict_CheckExact(od)) + res = PyDict_DelItem(od, key); + else + res = PyMapping_DelItem(od, key); + if (res >= 0) { + /* node may be out of date, so refresh it. */ + node = _odict_find_node((PyODictObject *)od, key); + if (node == NULL && PyErr_Occurred()) + res = -1; + } + if (res == -1) { + Py_DECREF(item); + return NULL; + } + if (node != NULL && _odict_clear_node((PyODictObject *)od, node, key) < 0) + return NULL; + + return item; +} + +/* keys() */ + +/* MutableMapping.keys() does not have a docstring. */ +PyDoc_STRVAR(odict_keys__doc__, ""); + +static PyObject * odictkeys_new(PyObject *od); /* forward */ + +/* values() */ + +/* MutableMapping.values() does not have a docstring. */ +PyDoc_STRVAR(odict_values__doc__, ""); + +static PyObject * odictvalues_new(PyObject *od); /* forward */ + +/* items() */ + +/* MutableMapping.items() does not have a docstring. */ +PyDoc_STRVAR(odict_items__doc__, ""); + +static PyObject * odictitems_new(PyObject *od); /* forward */ + +/* update() */ + +/* MutableMapping.update() does not have a docstring. */ +PyDoc_STRVAR(odict_update__doc__, ""); + +/* forward */ +static PyObject * mutablemapping_update(PyObject *, PyObject *, PyObject *); + +#define odict_update mutablemapping_update + +/* clear() */ + +PyDoc_STRVAR(odict_clear__doc__, + "od.clear() -> None. Remove all items from od."); + +static PyObject * +odict_clear(register PyODictObject *od) +{ + PyDict_Clear((PyObject *)od); + _odict_clear_nodes(od, 1); + Py_RETURN_NONE; +} + +/* copy() */ + +PyDoc_STRVAR(odict_copy__doc__, "od.copy() -> a shallow copy of od"); + +static PyObject * +odict_copy(register PyODictObject *od) +{ + _ODictNode *node; + PyObject *od_copy; + + if (PyODict_CheckExact(od)) + od_copy = PyODict_New(); + else + od_copy = PyObject_CallFunctionObjArgs((PyObject *)Py_TYPE(od), NULL); + if (od_copy == NULL) + return NULL; + + /* XXX It is possible that the linked-list could change during the loops. */ + if (PyODict_CheckExact(od)) { + _odict_FOREACH(od, node) { + int res = PyODict_SetItem((PyObject *)od_copy, + _odictnode_KEY(node), + _odictnode_VALUE(node, od)); + if (res != 0) + goto fail; + } + } + else { + _odict_FOREACH(od, node) { + int res; + PyObject *value = PyObject_GetItem((PyObject *)od, + _odictnode_KEY(node)); + if (value == NULL) + goto fail; + res = PyObject_SetItem((PyObject *)od_copy, + _odictnode_KEY(node), value); + Py_DECREF(value); + if (res != 0) + goto fail; + } + } + return od_copy; + +fail: + Py_DECREF(od_copy); + return NULL; +} + +/* __reversed__() */ + +PyDoc_STRVAR(odict_reversed__doc__, "od.__reversed__() <==> reversed(od)"); + +static PyObject * odict_reversed(PyODictObject *); /* forward */ + + +/* move_to_end() */ + +PyDoc_STRVAR(odict_move_to_end__doc__, +"Move an existing element to the end (or beginning if last==False).\n\ +\n\ + Raises KeyError if the element does not exist.\n\ + When last=True, acts like a fast version of self[key]=self.pop(key).\n\ +\n\ + "); + +static PyObject * +odict_move_to_end(PyODictObject *od, PyObject *args) +{ + PyObject *key, *last = NULL; + Py_ssize_t pos = -1; + + /* both borrowed */ + if (!PyArg_UnpackTuple(args, "move_to_end", 1, 2, &key, &last)) + return NULL; + if (_odict_EMPTY(od)) { + PyErr_SetObject(PyExc_KeyError, key); + return NULL; + } + if (last != NULL) { + Py_INCREF(last); + pos = PyObject_IsTrue(last) ? -1 : 0; + } + Py_INCREF(key); + if (pos == 0) { + /* Only move if not already the first one. */ + PyObject *first_key = _odictnode_KEY(_odict_FIRST(od)); + if (PyObject_RichCompareBool(key, first_key, Py_NE)) { + _ODictNode *node = _odict_pop_node(od, NULL, key); + if (node != NULL) { + _odict_add_head(od, node); + } + else { + if (!PyErr_Occurred()) + PyErr_SetObject(PyExc_KeyError, key); + return NULL; + } + } + } + else if (pos == -1) { + /* Only move if not already the last one. */ + PyObject *last_key = _odictnode_KEY(_odict_LAST(od)); + if (PyObject_RichCompareBool(key, last_key, Py_NE)) { + _ODictNode *node = _odict_pop_node(od, NULL, key); + if (node != NULL) { + _odict_add_tail(od, node); + } + else { + if (!PyErr_Occurred()) + PyErr_SetObject(PyExc_KeyError, key); + return NULL; + } + } + } + Py_DECREF(key); + Py_XDECREF(last); + Py_RETURN_NONE; +} + + +/* tp_methods */ + +static PyMethodDef odict_methods[] = { + + /* explicitly defined so we can align docstrings with + * collections.OrderedDict */ + {"__delitem__", (PyCFunction)odict_mp_ass_sub, METH_NOARGS, + odict_delitem__doc__}, + {"__eq__", (PyCFunction)odict_eq, METH_NOARGS, + odict_eq__doc__}, + {"__init__", (PyCFunction)odict_init, METH_NOARGS, + odict_init__doc__}, + {"__iter__", (PyCFunction)odict_iter, METH_NOARGS, + odict_iter__doc__}, + {"__ne__", (PyCFunction)odict_ne, METH_NOARGS, + odict_ne__doc__}, + {"__repr__", (PyCFunction)odict_repr, METH_NOARGS, + odict_repr__doc__}, + {"__setitem__", (PyCFunction)odict_mp_ass_sub, METH_NOARGS, + odict_setitem__doc__}, + {"fromkeys", (PyCFunction)odict_fromkeys, METH_VARARGS | METH_CLASS, + odict_fromkeys__doc__}, + + /* overridden dict methods */ + {"__sizeof__", (PyCFunction)odict_sizeof, METH_NOARGS, + odict_sizeof__doc__}, + {"__reduce__", (PyCFunction)odict_reduce, METH_NOARGS, + odict_reduce__doc__}, + {"setdefault", (PyCFunction)odict_setdefault, METH_VARARGS, + odict_setdefault__doc__}, + {"pop", (PyCFunction)odict_pop, METH_VARARGS, + odict_pop__doc__}, + {"popitem", (PyCFunction)odict_popitem, METH_VARARGS, + odict_popitem__doc__}, + {"keys", (PyCFunction)odictkeys_new, METH_NOARGS, + odict_keys__doc__}, + {"values", (PyCFunction)odictvalues_new, METH_NOARGS, + odict_values__doc__}, + {"items", (PyCFunction)odictitems_new, METH_NOARGS, + odict_items__doc__}, + {"update", (PyCFunction)odict_update, METH_VARARGS | METH_KEYWORDS, + odict_update__doc__}, + {"clear", (PyCFunction)odict_clear, METH_NOARGS, + odict_clear__doc__}, + {"copy", (PyCFunction)odict_copy, METH_NOARGS, + odict_copy__doc__}, + + /* new methods */ + {"__reversed__", (PyCFunction)odict_reversed, METH_NOARGS, + odict_reversed__doc__}, + {"move_to_end", (PyCFunction)odict_move_to_end, METH_VARARGS, + odict_move_to_end__doc__}, + + {NULL, NULL} /* sentinel */ +}; + + +/* ---------------------------------------------- + * OrderedDict members + */ + +/* tp_members */ + +static PyMemberDef odict_members[] = { + {"__dict__", T_OBJECT, offsetof(PyODictObject, od_inst_dict), READONLY}, + {0} +}; + + +/* ---------------------------------------------- + * OrderedDict type slot methods + */ + +/* tp_dealloc */ + +static void +odict_dealloc(PyODictObject *self) +{ + PyObject_GC_UnTrack(self); + Py_TRASHCAN_SAFE_BEGIN(self); + Py_XDECREF(self->od_inst_dict); + if (self->od_weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *)self); + + _odict_clear_nodes(self, 0); + Py_TRASHCAN_SAFE_END(self); + + /* must be last */ + PyDict_Type.tp_dealloc((PyObject *)self); +}; + +/* tp_repr */ + +static PyObject * +odict_repr(PyODictObject *self) +{ + int i; + const char *formatstr; + _Py_IDENTIFIER(__class__); + _Py_IDENTIFIER(__name__); + Py_ssize_t count = -1; + PyObject *pieces = NULL, *result = NULL, *cls = NULL; + PyObject *classname = NULL, *format = NULL, *args = NULL; + _ODictNode *node; + + i = Py_ReprEnter((PyObject *)self); + if (i != 0) { + return i > 0 ? PyUnicode_FromString("...") : NULL; + } + + if (PyODict_SIZE(self) == 0) { + /* "OrderedDict()" */ + goto Finish; + } + + if (PyODict_CheckExact(self)) { + pieces = PyList_New(PyODict_SIZE(self)); + if (pieces == NULL) + goto Done; + + _odict_FOREACH(self, node) { + PyObject *pair = PyTuple_Pack(2, _odictnode_KEY(node), + _odictnode_VALUE(node, self)); + if (pair == NULL) + goto Done; + + PyList_SET_ITEM(pieces, ++count, pair); /* steals reference */ + } + } + else { + PyObject *items = PyObject_CallMethod((PyObject *)self, "items", NULL); + if (items == NULL) + goto Done; + pieces = PySequence_List(items); + Py_DECREF(items); + if(pieces == NULL) + goto Done; + } + +Finish: + cls = _PyObject_GetAttrId((PyObject *)self, &PyId___class__); + if (cls == NULL) + goto Done; + classname = _PyObject_GetAttrId(cls, &PyId___name__); + if (classname == NULL) + goto Done; + + if (pieces == NULL) { + formatstr = "%s()"; + args = PyTuple_Pack(1, classname); + } + else { + formatstr = "%s(%r)"; + args = PyTuple_Pack(2, classname, pieces); + } + if (args == NULL) + goto Done; + + format = PyUnicode_InternFromString(formatstr); + if (format == NULL) + goto Done; + + result = PyUnicode_Format(format, args); +Done: + Py_XDECREF(pieces); + Py_XDECREF(classname); + Py_XDECREF(format); + Py_XDECREF(args); + Py_ReprLeave((PyObject *)self); + return result; +}; + +/* tp_doc */ + +PyDoc_STRVAR(odict_doc, + "Dictionary that remembers insertion order"); + +/* tp_traverse */ + +static int +odict_traverse(PyODictObject *od, visitproc visit, void *arg) +{ + Py_VISIT(od->od_inst_dict); + Py_VISIT(od->od_weakreflist); + return PyDict_Type.tp_traverse((PyObject *)od, visit, arg); +} + +/* tp_clear */ + +static int +odict_tp_clear(PyODictObject *od) +{ + Py_CLEAR(od->od_inst_dict); + Py_CLEAR(od->od_weakreflist); + odict_clear(od); + return 0; +} + +/* tp_richcompare */ + +static PyObject * +odict_richcompare(PyObject *v, PyObject *w, int op) +{ + /* XXX allow for any mapping type? */ + if (!PyODict_Check(v) || !PyDict_Check(w)) { + Py_RETURN_NOTIMPLEMENTED; + } + else if (op == Py_EQ || op == Py_NE) { + PyObject *res, *cmp; + int eq; + + cmp = PyDict_Type.tp_richcompare(v, w, op); + + if (cmp == NULL) + eq = -1; + else if (cmp == Py_False) + eq = 0; + else if (!PyODict_Check(w)) + eq = 1; + else { + /* Try comparing odict keys. */ + eq = _odict_keys_equal((PyODictObject *)v, (PyODictObject *)w); + } + Py_DECREF(cmp); + + if (eq < 0) + res = NULL; + else { + res = (eq == (op == Py_EQ)) ? Py_True : Py_False; + Py_INCREF(res); + } + return res; + } + else { + Py_RETURN_NOTIMPLEMENTED; + } +}; + +/* tp_iter */ + +static PyObject * odictiter_new(PyODictObject *, PyTypeObject *, int); /* forward */ + +static PyObject * +odict_iter(PyODictObject *od) +{ + return odictiter_new(od, &PyODictIterKey_Type, 0); +}; + +/* tp_init */ + +static int +odict_init(PyObject *self, PyObject *args, PyObject *kwds) +{ + Py_ssize_t len = PyObject_Length(args); + + if (len == -1) + return -1; + if (len > 1) { + char *msg = "expected at most 1 arguments, got %d"; + PyErr_Format(PyExc_TypeError, msg, len); + return -1; + } + + /* __init__() triggering update() is just the way things are! */ + if (odict_update(self, args, kwds) == NULL) + return -1; + else + return 0; +}; + +/* tp_new */ + +static PyObject * +odict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *od = PyDict_Type.tp_new(type, args, kwds); + if (od != NULL) { + ((PyODictObject *)od)->od_inst_dict = PyDict_New(); + ((PyODictObject *)od)->od_weakreflist = NULL; + if (_odict_initialize((PyODictObject *)od) < 0) + return NULL; + } + return od; +} + +/* PyODict_Type */ + +PyTypeObject PyODict_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "collections.OrderedDict", /* tp_name */ + sizeof(PyODictObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)odict_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)odict_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + &odict_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + odict_doc, /* tp_doc */ + (traverseproc)odict_traverse, /* tp_traverse */ + (inquiry)odict_tp_clear, /* tp_clear */ + (richcmpfunc)odict_richcompare, /* tp_richcompare */ + offsetof(PyODictObject, od_weakreflist), /* tp_weaklistoffset */ + (getiterfunc)odict_iter, /* tp_iter */ + 0, /* tp_iternext */ + odict_methods, /* tp_methods */ + odict_members, /* tp_members */ + 0, /* tp_getset */ + &PyDict_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(PyODictObject, od_inst_dict), /* tp_dictoffset */ + (initproc)odict_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + (newfunc)odict_new, /* tp_new */ + 0, /* tp_free */ +}; + + +/* ---------------------------------------------- + * the public OrderedDict API + */ + +PyObject * +PyODict_New(void) { + return odict_new(&PyODict_Type, NULL, NULL); +}; + +int +PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) { + int res = PyDict_SetItem(od, key, value); + if (res == 0) { + res = _odict_add_new_node((PyODictObject *)od, key); + } + return res; +}; + +int +PyODict_DelItem(PyObject *od, PyObject *key) { + int res = PyDict_DelItem(od, key); + if (res == 0) + if (_odict_clear_node((PyODictObject *)od, NULL, key) < 0) + return -1; + return res; +}; + + +/* ------------------------------------------- + * The OrderedDict views (keys/values/items) + */ + +typedef struct { + PyObject_HEAD + PyODictObject *di_odict; + PyObject *di_current; + PyObject *di_result; /* reusable result tuple for iteritems */ +} odictiterobject; + +static PyObject * +odictiter_new(PyODictObject *od, PyTypeObject *itertype, int reverse) +{ + odictiterobject *di; + _ODictNode *node; + + di = PyObject_GC_New(odictiterobject, itertype); + if (di == NULL) + return NULL; + + Py_INCREF(od); + di->di_odict = od; + node = reverse ? _odict_LAST(od) : _odict_FIRST(od); + di->di_current = node ? _odictnode_KEY(node) : NULL; + Py_XINCREF(di->di_current); + + if (itertype == &PyODictIterItem_Type) { + di->di_result = PyTuple_Pack(2, Py_None, Py_None); + if (di->di_result == NULL) { + Py_DECREF(di); + return NULL; + } + } + else + di->di_result = NULL; + _PyObject_GC_TRACK(di); + return (PyObject *)di; +} + +static void +odictiter_dealloc(odictiterobject *di) +{ + _PyObject_GC_UNTRACK(di); + Py_DECREF(di->di_odict); + Py_XDECREF(di->di_current); + if (Py_TYPE(di) == &PyODictIterItem_Type) { + Py_DECREF(di->di_result); + } + PyObject_GC_Del(di); +} + +static int +odictiter_traverse(odictiterobject *di, visitproc visit, void *arg) +{ + Py_VISIT(di->di_odict); + Py_VISIT(di->di_current); /* A key could be any type, not just str. */ + Py_VISIT(di->di_result); + return 0; +} + +/* No need for tp_clear because odictiterobject is not mutable. */ + +static PyObject * +odictiter_advance(odictiterobject *di, int reversed) +{ + PyObject *key; + _ODictNode *node; + + if (di->di_current == NULL) + return NULL; + node = _odict_find_node(di->di_odict, di->di_current); + if (node == NULL) { + /* Must have been deleted. */ + Py_DECREF(di->di_current); + di->di_current = NULL; + return NULL; + } + key = di->di_current; + node = reversed ? _odictnode_PREV(node) : _odictnode_NEXT(node); + if (node == NULL) { + /* Reached the end. */ + Py_DECREF(di->di_current); + di->di_current = NULL; + } + else { + di->di_current = _odictnode_KEY(node); + Py_INCREF(di->di_current); + } + Py_INCREF(key); /* XXX Superfluous? */ + return key; +} + +static PyObject * odictiter_iternextkey(odictiterobject *); /* forward */ +static PyObject * odictiter_iternextitem(odictiterobject *); /* forward */ +static PyObject * odictiter_iternextvalue(odictiterobject *); /* forward */ + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling"); + +static PyObject * +odictiter_reduce(odictiterobject *di) +{ + PyObject *list; + + list = PyList_New(0); + if (!list) + return NULL; + + /* iterate the temporary into a list */ + for(;;) { + PyObject *element = NULL; + if (Py_TYPE(di) == &PyODictIterItem_Type) + element = odictiter_iternextitem(di); + else if (Py_TYPE(di) == &PyODictIterKey_Type) + element = odictiter_iternextkey(di); + else if (Py_TYPE(di) == &PyODictIterValue_Type) + element = odictiter_iternextvalue(di); + else + assert(0); + if (element) { + if (PyList_Append(list, element)) { + Py_DECREF(element); + Py_DECREF(list); + return NULL; + } + Py_DECREF(element); + } + else { + /* done iterating? */ + break; + } + } + if (PyErr_Occurred()) { + Py_DECREF(list); + return NULL; + } + return Py_BuildValue("N(N)", _PyObject_GetBuiltin("iter"), list); +} + +static PyMethodDef odictiter_methods[] = { + {"__reduce__", (PyCFunction)odictiter_reduce, METH_NOARGS, + reduce_doc}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyODictIterView_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "odict_viewiterator", /* tp_name */ + sizeof(odictiterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)odictiter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)odictiter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + 0, /* tp_iternext */ + odictiter_methods, /* tp_methods */ + 0, +}; + +/* keys() */ + +static PyObject * +odictiter_iternextkey(odictiterobject *di) +{ + return odictiter_advance(di, 0); +} + +PyTypeObject PyODictIterKey_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "odict_keyiterator", /* tp_name */ + 0, /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + (iternextfunc)odictiter_iternextkey, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyODictIterView_Type, /* tp_base */ + 0, +}; + +static PyObject * +odictkeys_iter(_PyDictViewObject *dv) +{ + if (dv->dv_dict == NULL) { + Py_RETURN_NONE; + } + return odictiter_new((PyODictObject *)dv->dv_dict, &PyODictIterKey_Type, 0); +} + +PyTypeObject PyODictKeys_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "odict_keys", /* tp_name */ + 0, /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + (getiterfunc)odictkeys_iter, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyDictKeys_Type, /* tp_base */ +}; + +static PyObject * +odictkeys_new(PyObject *od) +{ + return _PyDictView_New(od, &PyODictKeys_Type); +} + +/* items() */ + +static PyObject * +odictiter_iternextitem(odictiterobject *di) +{ + PyObject *key, *value, *result = di->di_result; + + key = odictiter_advance(di, 0); /* new reference */ + if (key == NULL) + return NULL; + value = PyODict_GetItem((PyObject *)di->di_odict, key); /* borrowed */ + + if (result && result->ob_refcnt == 1) { + /* not in use so we can reuse it--the common case during iteration */ + Py_INCREF(result); + Py_DECREF(PyTuple_GET_ITEM(result, 0)); /* borrowed */ + Py_DECREF(PyTuple_GET_ITEM(result, 1)); /* borrowed */ + } + else { + result = PyTuple_New(2); + if (result == NULL) + return NULL; + } + + Py_INCREF(value); + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ + + return result; +} + +PyTypeObject PyODictIterItem_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "odict_itemiterator", /* tp_name */ + 0, /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + (iternextfunc)odictiter_iternextitem, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyODictIterView_Type, /* tp_base */ + 0, +}; + +static PyObject * +odictitems_iter(_PyDictViewObject *dv) +{ + if (dv->dv_dict == NULL) { + Py_RETURN_NONE; + } + return odictiter_new((PyODictObject *)dv->dv_dict, &PyODictIterItem_Type, 0); +} + +PyTypeObject PyODictItems_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "odict_items", /* tp_name */ + 0, /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + (getiterfunc)odictitems_iter, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyDictItems_Type, /* tp_base */ +}; + +static PyObject * +odictitems_new(PyObject *od) +{ + return _PyDictView_New(od, &PyODictItems_Type); +} + +/* values() */ + +static PyObject * +odictiter_iternextvalue(odictiterobject *di) +{ + PyObject *key, *value; + + key = odictiter_advance(di, 0); + if (key == NULL) + return NULL; + value = PyODict_GetItem((PyObject *)di->di_odict, key); + Py_XINCREF(value); + Py_DECREF(key); + return value; +} + +PyTypeObject PyODictIterValue_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "odict_valueiterator", /* tp_name */ + 0, /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + (iternextfunc)odictiter_iternextvalue, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyODictIterView_Type, /* tp_base */ + 0, +}; + +static PyObject * +odictvalues_iter(_PyDictViewObject *dv) +{ + if (dv->dv_dict == NULL) { + Py_RETURN_NONE; + } + return odictiter_new((PyODictObject *)dv->dv_dict, &PyODictIterValue_Type, 0); +} + +PyTypeObject PyODictValues_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "odict_values", /* tp_name */ + 0, /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + (getiterfunc)odictvalues_iter, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyDictValues_Type, /* tp_base */ +}; + +static PyObject * +odictvalues_new(PyObject *od) +{ + return _PyDictView_New(od, &PyODictValues_Type); +} + +/* __reverse__() */ + +static PyObject * +odictiter_iternextreversed(odictiterobject *di) +{ + return odictiter_advance(di, 1); +} + +PyTypeObject PyODictIterReversed_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "odict_reversediterator", /* tp_name */ + 0, /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + (iternextfunc)odictiter_iternextreversed, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyODictIterView_Type, /* tp_base */ + 0, +}; + +static PyObject * +odict_reversed(PyODictObject *od) +{ + if (_odict_EMPTY(od)) { + Py_RETURN_NONE; + } + return odictiter_new(od, &PyODictIterReversed_Type, 1); +} + + +/* ---------------------------------------------- + MutableMappping implementations + +Mapping: + +============ =========== +method uses +============ =========== +__contains__ __getitem__ +__eq__ items +__getitem__ + +__iter__ + +__len__ + +__ne__ __eq__ +get __getitem__ +items ItemsView +keys KeysView +values ValuesView +============ =========== + +ItemsView uses __len__, __iter__, and __getitem__. +KeysView uses __len__, __iter__, and __contains__. +ValuesView uses __len__, __iter__, and __getitem__. + +MutableMapping: + +============ =========== +method uses +============ =========== +__delitem__ + +__setitem__ + +clear popitem +pop __getitem__ + __delitem__ +popitem __iter__ + _getitem__ + __delitem__ +setdefault __getitem__ + __setitem__ +update __setitem__ +============ =========== +*/ + +/* XXX move to abstract.c? */ + +static int +mutablemapping_add_pairs(PyObject *self, PyObject *pairs) +{ + PyObject *pair, *iterator; + int res = 0; + + iterator = PyObject_GetIter(pairs); + if (iterator == NULL) + return -1; + PyErr_Clear(); + + while ((pair = PyIter_Next(iterator)) != NULL) { + /* XXX could be more efficient (see UNPACK_SEQUENCE in ceval.c) */ + PyObject * key, *value = NULL; + PyObject *pair_iterator = PyObject_GetIter(pair); + + key = PyIter_Next(pair_iterator); + if (key == NULL) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ValueError, + "need more than 0 values to unpack"); + goto Done; + } + + value = PyIter_Next(pair_iterator); + if (value == NULL) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ValueError, + "need more than 1 value to unpack"); + goto Done; + } + + if (PyIter_Next(pair_iterator) != NULL) { + PyErr_SetString(PyExc_ValueError, + "too many values to unpack (expected 2)"); + goto Done; + } + else if (PyErr_Occurred()) + goto Done; + + res = PyObject_SetItem(self, key, value); + +Done: + Py_DECREF(pair); + Py_DECREF(pair_iterator); + Py_XDECREF(key); + Py_XDECREF(value); + if (PyErr_Occurred()) + break; + } + Py_DECREF(iterator); + + if (res < 0 || PyErr_Occurred() != NULL) + return -1; + else + return 0; +} + +static PyObject * +mutablemapping_update(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int res = 0; + Py_ssize_t len = (args != NULL) ? PyObject_Size(args) : 0; + + /* first handle args, if any */ + if (len < 0) + return NULL; + else if (len > 1) { + char *msg = "update() takes at most 1 positional argument (%d given)"; + PyErr_Format(PyExc_TypeError, msg, len); + return NULL; + } + else if (len == 1) { + PyObject *other = PyTuple_GET_ITEM(args, 0); /* borrowed reference */ + if (other == NULL) + return NULL; + Py_INCREF(other); + if (PyObject_HasAttrString(other, "items")) { + PyObject *items = PyMapping_Items(other); + Py_DECREF(other); + if (items == NULL) + return NULL; + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + if (res == -1) + return NULL; + } + else if (PyObject_HasAttrString(other, "keys")) { + PyObject *keys, *iterator, *key; + keys = PyObject_CallMethod(other, "keys", NULL); + Py_DECREF(other); + if (keys == NULL) + return NULL; + iterator = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iterator == NULL) + return NULL; + while (res == 0 && (key = PyIter_Next(iterator))) { + PyObject *value = PyObject_GetItem(other, key); + if (value != NULL) { + res = PyObject_SetItem(self, key, value); + Py_DECREF(value); + } + else { + res = -1; + } + Py_DECREF(key); + } + Py_DECREF(iterator); + if (res != 0 || PyErr_Occurred()) + return NULL; + } + else { + res = mutablemapping_add_pairs(self, other); + Py_DECREF(other); + if (res != 0) + return NULL; + } + } + + /* now handle kwargs */ + len = (kwargs != NULL) ? PyObject_Size(kwargs) : 0; + if (len < 0) + return NULL; + else if (len > 0) { + PyObject *items; + if (!PyMapping_Check(kwargs)) { + PyErr_SetString(PyExc_TypeError, "expected mapping for kwargs"); + return NULL; + } + items = PyMapping_Items(kwargs); + if (items == NULL) + return NULL; + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + if (res == -1) + return NULL; + } + + Py_RETURN_NONE; +}