Index: Python/ceval.c =================================================================== --- Python/ceval.c (revision 46513) +++ Python/ceval.c (working copy) @@ -1855,15 +1855,26 @@ long hash = ((PyStringObject *)w)->ob_shash; if (hash != -1) { PyDictObject *d; + PyDictEntry *e; d = (PyDictObject *)(f->f_globals); - x = d->ma_lookup(d, w, hash)->me_value; + e = d->ma_lookup(d, w, hash); + if (e == NULL) { + x = NULL; + break; + } + x = e->me_value; if (x != NULL) { Py_INCREF(x); PUSH(x); continue; } d = (PyDictObject *)(f->f_builtins); - x = d->ma_lookup(d, w, hash)->me_value; + e = d->ma_lookup(d, w, hash); + if (e == NULL) { + x = NULL; + break; + } + x = e->me_value; if (x != NULL) { Py_INCREF(x); PUSH(x); Index: Objects/dictobject.c =================================================================== --- Objects/dictobject.c (revision 46513) +++ Objects/dictobject.c (working copy) @@ -231,10 +231,7 @@ register unsigned int mask = mp->ma_mask; dictentry *ep0 = mp->ma_table; register dictentry *ep; - register int restore_error; - register int checked_error; register int cmp; - PyObject *err_type, *err_value, *err_tb; PyObject *startkey; i = hash & mask; @@ -242,24 +239,17 @@ if (ep->me_key == NULL || ep->me_key == key) return ep; - restore_error = checked_error = 0; if (ep->me_key == dummy) freeslot = ep; else { if (ep->me_hash == hash) { - /* error can't have been checked yet */ - checked_error = 1; - if (PyErr_Occurred()) { - restore_error = 1; - PyErr_Fetch(&err_type, &err_value, &err_tb); - } startkey = ep->me_key; cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); if (cmp < 0) - PyErr_Clear(); + return NULL; if (ep0 == mp->ma_table && ep->me_key == startkey) { if (cmp > 0) - goto Done; + return ep; } else { /* The compare did major nasty stuff to the @@ -267,8 +257,7 @@ * XXX A clever adversary could prevent this * XXX from terminating. */ - ep = lookdict(mp, key, hash); - goto Done; + return lookdict(mp, key, hash); } } freeslot = NULL; @@ -279,29 +268,18 @@ for (perturb = hash; ; perturb >>= PERTURB_SHIFT) { i = (i << 2) + i + perturb + 1; ep = &ep0[i & mask]; - if (ep->me_key == NULL) { - if (freeslot != NULL) - ep = freeslot; - break; - } + if (ep->me_key == NULL) + return freeslot == NULL ? ep : freeslot; if (ep->me_key == key) - break; + return ep; if (ep->me_hash == hash && ep->me_key != dummy) { - if (!checked_error) { - checked_error = 1; - if (PyErr_Occurred()) { - restore_error = 1; - PyErr_Fetch(&err_type, &err_value, - &err_tb); - } - } startkey = ep->me_key; cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); if (cmp < 0) - PyErr_Clear(); + return NULL; if (ep0 == mp->ma_table && ep->me_key == startkey) { if (cmp > 0) - break; + return ep; } else { /* The compare did major nasty stuff to the @@ -309,18 +287,12 @@ * XXX A clever adversary could prevent this * XXX from terminating. */ - ep = lookdict(mp, key, hash); - break; + return lookdict(mp, key, hash); } } else if (ep->me_key == dummy && freeslot == NULL) freeslot = ep; } - -Done: - if (restore_error) - PyErr_Restore(err_type, err_value, err_tb); - return ep; } /* @@ -390,7 +362,7 @@ Used both by the internal resize routine and by the public insert routine. Eats a reference to key and one to value. */ -static void +static int insertdict(register dictobject *mp, PyObject *key, long hash, PyObject *value) { PyObject *old_value; @@ -399,6 +371,11 @@ assert(mp->ma_lookup != NULL); ep = mp->ma_lookup(mp, key, hash); + if (ep == NULL) { + Py_DECREF(key); + Py_DECREF(value); + return -1; + } if (ep->me_value != NULL) { old_value = ep->me_value; ep->me_value = value; @@ -417,9 +394,39 @@ ep->me_value = value; mp->ma_used++; } + return 0; } /* +Internal routine used by dictresize() to insert an item which is +known to be absent from the dict. This routine also assumes that +the dict contains no deleted entries. Besides the performance benefit, +using insertdict() in dictresize() is dangerous (SF bug #1456209). +*/ +static void +insertdict_clean(register dictobject *mp, PyObject *key, long hash, + PyObject *value) +{ + register Py_ssize_t i; + register size_t perturb; + register unsigned int mask = mp->ma_mask; + dictentry *ep0 = mp->ma_table; + register dictentry *ep; + + i = hash & mask; + ep = &ep0[i]; + for (perturb = hash; ep->me_key != NULL; perturb >>= PERTURB_SHIFT) { + i = (i << 2) + i + perturb + 1; + ep = &ep0[i & mask]; + } + mp->ma_fill++; + ep->me_key = key; + ep->me_hash = hash; + ep->me_value = value; + mp->ma_used++; +} + +/* Restructure the table by allocating a new table and reinserting all items again. When entries have been deleted, the new table may actually be smaller than the old one. @@ -491,7 +498,8 @@ for (ep = oldtable; i > 0; ep++) { if (ep->me_value != NULL) { /* active entry */ --i; - insertdict(mp, ep->me_key, ep->me_hash, ep->me_value); + insertdict_clean(mp, ep->me_key, ep->me_hash, + ep->me_value); } else if (ep->me_key != NULL) { /* dummy entry */ --i; @@ -511,6 +519,8 @@ { long hash; dictobject *mp = (dictobject *)op; + dictentry *ep; + PyThreadState *tstate; if (!PyDict_Check(op)) { return NULL; } @@ -523,7 +533,29 @@ return NULL; } } - return (mp->ma_lookup)(mp, key, hash)->me_value; + + /* We can arrive here with a NULL tstate during initialization: + try running "python -Wi" for an example related to string + interning. Let's just hope that no exception occurs then... */ + tstate = PyThreadState_GET(); + if (tstate != NULL && tstate->curexc_type != NULL) { + /* preserve the existing exception */ + PyObject *err_type, *err_value, *err_tb; + PyErr_Fetch(&err_type, &err_value, &err_tb); + ep = (mp->ma_lookup)(mp, key, hash); + /* ignore errors */ + PyErr_Restore(err_type, err_value, err_tb); + if (ep == NULL) + return NULL; + } + else { + ep = (mp->ma_lookup)(mp, key, hash); + if (ep == NULL) { + PyErr_Clear(); + return NULL; + } + } + return ep->me_value; } /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the @@ -558,7 +590,8 @@ n_used = mp->ma_used; Py_INCREF(value); Py_INCREF(key); - insertdict(mp, key, hash, value); + if (insertdict(mp, key, hash, value) != 0) + return -1; /* If we added a key, we can safely resize. Otherwise just return! * If fill >= 2/3 size, adjust size. Normally, this doubles or * quaduples the size, but it's also possible for the dict to shrink @@ -598,6 +631,8 @@ } mp = (dictobject *)op; ep = (mp->ma_lookup)(mp, key, hash); + if (ep == NULL) + return -1; if (ep->me_value == NULL) { PyErr_SetObject(PyExc_KeyError, key); return -1; @@ -882,6 +917,7 @@ { PyObject *v; long hash; + dictentry *ep; assert(mp->ma_table != NULL); if (!PyString_CheckExact(key) || (hash = ((PyStringObject *) key)->ob_shash) == -1) { @@ -889,7 +925,10 @@ if (hash == -1) return NULL; } - v = (mp->ma_lookup)(mp, key, hash) -> me_value; + ep = (mp->ma_lookup)(mp, key, hash); + if (ep == NULL) + return NULL; + v = ep->me_value; if (v == NULL) { if (!PyDict_CheckExact(mp)) { /* Look up __missing__ method if we're a subclass. */ @@ -1247,8 +1286,10 @@ PyDict_GetItem(a, entry->me_key) == NULL)) { Py_INCREF(entry->me_key); Py_INCREF(entry->me_value); - insertdict(mp, entry->me_key, entry->me_hash, - entry->me_value); + if (insertdict(mp, entry->me_key, + entry->me_hash, + entry->me_value) != 0) + return -1; } } } @@ -1555,13 +1596,17 @@ { long hash; register long ok; + dictentry *ep; if (!PyString_CheckExact(key) || (hash = ((PyStringObject *) key)->ob_shash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; } - ok = (mp->ma_lookup)(mp, key, hash)->me_value != NULL; + ep = (mp->ma_lookup)(mp, key, hash); + if (ep == NULL) + return NULL; + ok = ep->me_value != NULL; return PyBool_FromLong(ok); } @@ -1572,6 +1617,7 @@ PyObject *failobj = Py_None; PyObject *val = NULL; long hash; + dictentry *ep; if (!PyArg_UnpackTuple(args, "get", 1, 2, &key, &failobj)) return NULL; @@ -1582,8 +1628,10 @@ if (hash == -1) return NULL; } - val = (mp->ma_lookup)(mp, key, hash)->me_value; - + ep = (mp->ma_lookup)(mp, key, hash); + if (ep == NULL) + return NULL; + val = ep->me_value; if (val == NULL) val = failobj; Py_INCREF(val); @@ -1598,6 +1646,7 @@ PyObject *failobj = Py_None; PyObject *val = NULL; long hash; + dictentry *ep; if (!PyArg_UnpackTuple(args, "setdefault", 1, 2, &key, &failobj)) return NULL; @@ -1608,7 +1657,10 @@ if (hash == -1) return NULL; } - val = (mp->ma_lookup)(mp, key, hash)->me_value; + ep = (mp->ma_lookup)(mp, key, hash); + if (ep == NULL) + return NULL; + val = ep->me_value; if (val == NULL) { val = failobj; if (PyDict_SetItem((PyObject*)mp, key, failobj)) @@ -1652,6 +1704,8 @@ return NULL; } ep = (mp->ma_lookup)(mp, key, hash); + if (ep == NULL) + return NULL; if (ep->me_value == NULL) { if (deflt) { Py_INCREF(deflt); @@ -1871,6 +1925,7 @@ { long hash; dictobject *mp = (dictobject *)op; + dictentry *ep; if (!PyString_CheckExact(key) || (hash = ((PyStringObject *) key)->ob_shash) == -1) { @@ -1878,7 +1933,10 @@ if (hash == -1) return -1; } - return (mp->ma_lookup)(mp, key, hash)->me_value != NULL; + ep = (mp->ma_lookup)(mp, key, hash); + if (ep == NULL) + return -1; + return ep->me_value != NULL; } /* Hack to implement "key in dict" */ Index: Lib/test/crashers/dictresize_attack.py =================================================================== --- Lib/test/crashers/dictresize_attack.py (revision 46513) +++ Lib/test/crashers/dictresize_attack.py (working copy) @@ -1,32 +0,0 @@ -# http://www.python.org/sf/1456209 - -# A dictresize() attack. If oldtable == mp->ma_smalltable then pure -# Python code can mangle with mp->ma_smalltable while it is being walked -# over. - -class X(object): - - def __hash__(self): - return 5 - - def __eq__(self, other): - if resizing: - d.clear() - return False - - -d = {} - -resizing = False - -d[X()] = 1 -d[X()] = 2 -d[X()] = 3 -d[X()] = 4 -d[X()] = 5 - -# now trigger a resize -resizing = True -d[9] = 6 - -# ^^^ I get Segmentation fault or Illegal instruction here. Index: Lib/test/test_operations.py =================================================================== --- Lib/test/test_operations.py (revision 46513) +++ Lib/test/test_operations.py (working copy) @@ -5,27 +5,16 @@ print 'XXX Mostly not yet implemented' -print '3.1 Dictionary lookups succeed even if __cmp__() raises an exception' +print '3.1 Dictionary lookups fail if __cmp__() raises an exception' -# SourceForge bug #112558: -# http://sourceforge.net/bugs/?func=detailbug&bug_id=112558&group_id=5470 - class BadDictKey: - already_printed_raising_error = 0 def __hash__(self): return hash(self.__class__) def __cmp__(self, other): if isinstance(other, self.__class__): - if not BadDictKey.already_printed_raising_error: - # How many times __cmp__ gets called depends on the hash - # code and the internals of the dict implementation; we - # know it will be called at least once, but that's it. - # already_printed_raising_error makes sure the expected- - # output file prints the msg at most once. - BadDictKey.already_printed_raising_error = 1 - print "raising error" + print "raising error" raise RuntimeError, "gotcha" return other @@ -33,9 +22,22 @@ x1 = BadDictKey() x2 = BadDictKey() d[x1] = 1 -d[x2] = 2 -print "No exception passed through." +for stmt in ['d[x2] = 2', + 'z = d[x2]', + 'x2 in d', + 'd.has_key(x2)', + 'd.get(x2)', + 'd.setdefault(x2, 42)', + 'd.pop(x2)', + 'd.update({x2: 2})']: + try: + exec stmt + except RuntimeError: + print "%s: caught the RuntimeError outside" % (stmt,) + else: + print "%s: No exception passed through!" # old CPython behavior + # Dict resizing bug, found by Jack Jansen in 2.2 CVS development. # This version got an assert failure in debug build, infinite loop in # release build. Unfortunately, provoking this kind of stuff requires @@ -50,3 +52,27 @@ del d[i] for i in range(5, 9): # i==8 was the problem d[i] = i + + +# Another dict resizing bug (SF bug #1456209). +# This caused Segmentation faults or Illegal instructions. + +class X(object): + def __hash__(self): + return 5 + def __eq__(self, other): + if resizing: + d.clear() + return False +d = {} +resizing = False +d[X()] = 1 +d[X()] = 2 +d[X()] = 3 +d[X()] = 4 +d[X()] = 5 +# now trigger a resize +resizing = True +d[9] = 6 + +print 'resize bugs not triggered.' Index: Lib/test/output/test_operations =================================================================== --- Lib/test/output/test_operations (revision 46513) +++ Lib/test/output/test_operations (working copy) @@ -1,6 +1,21 @@ test_operations 3. Operations XXX Mostly not yet implemented -3.1 Dictionary lookups succeed even if __cmp__() raises an exception +3.1 Dictionary lookups fail if __cmp__() raises an exception raising error -No exception passed through. +d[x2] = 2: caught the RuntimeError outside +raising error +z = d[x2]: caught the RuntimeError outside +raising error +x2 in d: caught the RuntimeError outside +raising error +d.has_key(x2): caught the RuntimeError outside +raising error +d.get(x2): caught the RuntimeError outside +raising error +d.setdefault(x2, 42): caught the RuntimeError outside +raising error +d.pop(x2): caught the RuntimeError outside +raising error +d.update({x2: 2}): caught the RuntimeError outside +resize bugs not triggered.