Index: Include/objimpl.h =================================================================== --- Include/objimpl.h (revision 85840) +++ Include/objimpl.h (working copy) @@ -321,7 +321,7 @@ #define Py_VISIT(op) \ do { \ if (op) { \ - int vret = visit((PyObject *)(op), arg); \ + int vret = visit((PyObject **)&(op), arg); \ if (vret) \ return vret; \ } \ Index: Include/object.h =================================================================== --- Include/object.h (revision 85840) +++ Include/object.h (working copy) @@ -194,7 +194,7 @@ /* End buffer interface */ typedef int (*objobjproc)(PyObject *, PyObject *); -typedef int (*visitproc)(PyObject *, void *); +typedef int (*visitproc)(PyObject **, void *); typedef int (*traverseproc)(PyObject *, visitproc, void *); typedef struct { Index: Objects/typeobject.c =================================================================== --- Objects/typeobject.c (revision 85840) +++ Objects/typeobject.c (working copy) @@ -738,7 +738,9 @@ char *addr = (char *)self + mp->offset; PyObject *obj = *(PyObject **)addr; if (obj != NULL) { - int err = visit(obj, arg); + /* Note that we must vist the actual addr for remapping + to work correctly */ + int err = visit((PyObject **)addr, arg); if (err) return err; } @@ -776,8 +778,10 @@ if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) /* For a heaptype, the instances count as references to the type. Traverse the type so the collector - can find cycles involving this link. */ - Py_VISIT(type); + can find cycles involving this link. + Note that we must vist the actual type reference + rather than use the local variable type */ + Py_VISIT(Py_TYPE(self)); if (basetraverse) return basetraverse(self, visit, arg); Index: Objects/dictobject.c =================================================================== --- Objects/dictobject.c (revision 85840) +++ Objects/dictobject.c (working copy) @@ -1004,6 +1004,35 @@ return 1; } +/* Internal version of PyDict_Next that returns the address of the key + and value. */ +int +_PyDict_Next_slot(PyObject *op, Py_ssize_t *ppos, + PyObject ***ppkey, PyObject ***ppvalue) +{ + register Py_ssize_t i; + register Py_ssize_t mask; + register PyDictEntry *ep; + + if (!PyDict_Check(op)) + return 0; + i = *ppos; + if (i < 0) + return 0; + ep = ((PyDictObject *)op)->ma_table; + mask = ((PyDictObject *)op)->ma_mask; + while (i <= mask && ep[i].me_value == NULL) + i++; + *ppos = i+1; + if (i > mask) + return 0; + if (ppkey) + *ppkey = &ep[i].me_key; + if (ppvalue) + *ppvalue = &ep[i].me_value; + return 1; +} + /* Methods */ static void @@ -1913,12 +1942,12 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) { Py_ssize_t i = 0; - PyObject *pk; - PyObject *pv; + PyObject **ppk; + PyObject **ppv; - while (PyDict_Next(op, &i, &pk, &pv)) { - Py_VISIT(pk); - Py_VISIT(pv); + while (_PyDict_Next_slot(op, &i, &ppk, &ppv)) { + Py_VISIT(*ppk); + Py_VISIT(*ppv); } return 0; } Index: Objects/genobject.c =================================================================== --- Objects/genobject.c (revision 85840) +++ Objects/genobject.c (working copy) @@ -10,7 +10,7 @@ static int gen_traverse(PyGenObject *gen, visitproc visit, void *arg) { - Py_VISIT((PyObject *)gen->gi_frame); + Py_VISIT(gen->gi_frame); Py_VISIT(gen->gi_code); return 0; } Index: Misc/NEWS =================================================================== --- Misc/NEWS (revision 85840) +++ Misc/NEWS (working copy) @@ -134,6 +134,8 @@ - Issue #9948: Fixed problem of losing filename case information. +- Issue #10194: Add gc.remap() method to change referencs to heap objects. + Extensions ---------- Index: Doc/library/gc.rst =================================================================== --- Doc/library/gc.rst (revision 85840) +++ Doc/library/gc.rst (working copy) @@ -128,6 +128,24 @@ from an argument, that integer object may or may not appear in the result list. +.. function:: remap(mapping) + + Modify all references to heap objects that appear as keys in mapping to + instead refer to the corresponding value. This function will only update + references in objects that support garbage collection; extension types + which refer to other objects but do not support garbage collection will + not be updated. Local variables in stack frames will be modified, but + active methods will not be updated. + + Care must be taken when using this function as it may modify otherwise + immutable objects such as tuples. If the hash value of a dictionary key + changes as the result of calling this function the dictionary will be + left in an invalid state. Avoid using :func:`remap` for any purpose + other than debugging. + + .. versionadded:: 3.2 + + .. function:: is_tracked(obj) Returns True if the object is currently tracked by the garbage collector, Index: Lib/test/test_gc.py =================================================================== --- Lib/test/test_gc.py (revision 85840) +++ Lib/test/test_gc.py (working copy) @@ -655,6 +655,185 @@ # empty __dict__. self.assertEqual(x, None) + +class GCRemapTests(unittest.TestCase): + def test_remap_inst(self): + class TestClass: + def __init__(self, name): + self.name = name + def __repr__(self): + return "TestClass('{0}')".format(self.name) + + class SlotsClass: + __slots__ = ['v'] + def __init__(self, v): + self.v = v + def __repr__(self): + return "SlotsClass({0})".format(repr(self.v)) + + a = TestClass('a') + b = TestClass('b') + self.assertIsNot(a, b) + + awr = weakref.ref(a) + alist = [a] + atuple = (a,) + adict_v = {'a': a} + adict_k = {a: 'a'} + aset = {a} + aslot = SlotsClass(a) + + mapping = {a: b} + gc.remap(mapping) + + # Verify mapping was left unchanged + t = mapping.popitem() + self.assertIsNot(t[0], t[1]) + self.assertIs(b, t[1]) + + # Verify that weakref is also unchanged + self.assertIsNot(b, awr()) + self.assertIs(t[0], awr()) + + # Verify other references were remapped + self.assertIs(a, b) + + self.assertIs(b, alist[0]) + self.assertIs(b, atuple[0]) + self.assertIs(b, adict_v["a"]) + self.assertIs(b, adict_k.popitem()[0]) + self.assertIs(b, aset.pop()) + self.assertIs(b, aslot.v) + + # Test swapping two instances + c = TestClass('c') + d = TestClass('d') + + cid = id(c) + did = id(d) + + mapping = {c: d, d: c} + gc.remap(mapping) + + self.assertIsNot(c, d) + self.assertEqual(cid, id(d)) + self.assertEqual(did, id(c)) + + def test_remap_class(self): + class TestClassA: + def __repr__(self): + return "TestClassA()" + + class TestClassB: + def __repr__(self): + return "TestClassB()" + + class SubClassA(TestClassA): + pass + + obj = TestClassA() + self.assertIsInstance(obj, TestClassA) + self.assertNotIsInstance(obj, TestClassB) + self.assertEqual(repr(obj), "TestClassA()") + self.assertFalse(issubclass(SubClassA, TestClassB)) + + mapping = {TestClassA: TestClassB} + gc.remap(mapping) + + self.assertIsInstance(obj, TestClassB) + self.assertEqual(repr(obj), "TestClassB()") + self.assertTrue(issubclass(SubClassA, TestClassB)) + + def test_remap_generator(self): + def generator_a(i): + # Guard against recursion if remapping is broken + if i > 5: + return + + yield i + for x in generator_a(i + 1): + yield x + yield i-1 + + def generator_b(i): + yield i + 10 + yield i + 11 + + mapping = {generator_a: generator_b} + + # Verify that nothing breaks if we remap a yielded + # generator, and that the old instance continues to run + # + # The remap is triggerd when two levels of generator + # are on the stack. + l = [x for x in generator_a(1) if x != 2 or not gc.remap(mapping)] + self.assertListEqual(l, [1, 2, 13, 14, 1, 0]) + + # Verify that the new generator works + l = [x for x in generator_a(1)] + self.assertListEqual(l, [11, 12]) + + def test_remap_ignore(self): + class TestClass: + def __init__(self, name): + self.name = name + def __repr__(self): + return "TestClass('{0}')".format(self.name) + + a = TestClass('a') + b = TestClass('b') + + atuple = (a,) + alist = [a] + aset = frozenset([a]) + + # Ignore the tuple and frozenset + mapping = {a: b} + ignore = {atuple, aset} + + gc.remap(mapping, ignore=ignore) + + oa, ob = dict(mapping).popitem() + self.assertIs(a, b) + self.assertIsNot(oa, ob) + self.assertIs(a, ob) + + # Ignored items not remapped + self.assertIs(atuple[0], oa) + self.assertIs(next(iter(aset)), oa) + # List was remapped + self.assertIs(alist[0], ob) + + # Now without ignoring + gc.remap(mapping) + + oa, ob = dict(mapping).popitem() + self.assertIs(atuple[0], ob) + self.assertIs(next(iter(aset)), ob) + + def test_remap_type_cache(self): + # Verify that the type cache is handled properly when + # remapping types. + + class BaseA: + def test(self): + return "a" + + class BaseB: + def test(self): + return "b" + + class Derived(BaseA): + pass + + obj = Derived() + self.assertEqual(obj.test(), "a") + + mapping = {BaseA: BaseB} + gc.remap(mapping) + + self.assertEqual(obj.test(), "b") + def test_main(): enabled = gc.isenabled() gc.disable() @@ -664,7 +843,7 @@ try: gc.collect() # Delete 2nd generation garbage - run_unittest(GCTests, GCTogglingTests) + run_unittest(GCTests, GCTogglingTests, GCRemapTests) finally: gc.set_debug(debug) # test gc.enable() even if GC is disabled by default Index: Lib/test/test_class.py =================================================================== --- Lib/test/test_class.py (revision 85840) +++ Lib/test/test_class.py (working copy) @@ -59,10 +59,11 @@ # "setattr", # "delattr", -callLst = [] +callLst = None def trackCall(f): def track(*args, **kwargs): - callLst.append((f.__name__, args)) + if callLst is not None: + callLst.append((f.__name__, args)) return f(*args, **kwargs) return track @@ -133,8 +134,13 @@ class ClassTests(unittest.TestCase): def setUp(self): - callLst[:] = [] + global callLst + callLst = [] + def tearDown(self): + global callLst + callLst = None + def assertCallStack(self, expected_calls): actualCallList = callLst[:] # need to copy because the comparison below will add # additional calls to callLst Index: Modules/_ctypes/_ctypes.c =================================================================== --- Modules/_ctypes/_ctypes.c (revision 85840) +++ Modules/_ctypes/_ctypes.c (working copy) @@ -2405,7 +2405,7 @@ PyCData_traverse(CDataObject *self, visitproc visit, void *arg) { Py_VISIT(self->b_objects); - Py_VISIT((PyObject *)self->b_base); + Py_VISIT(self->b_base); return 0; } Index: Modules/gcmodule.c =================================================================== --- Modules/gcmodule.c (revision 85840) +++ Modules/gcmodule.c (working copy) @@ -315,11 +315,11 @@ /* A traversal callback for subtract_refs. */ static int -visit_decref(PyObject *op, void *data) +visit_decref(PyObject **op, void *data) { - assert(op != NULL); - if (PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); + assert(*op != NULL); + if (PyObject_IS_GC(*op)) { + PyGC_Head *gc = AS_GC(*op); /* We're only interested in gc_refs for objects in the * generation being collected, which can be recognized * because only they have positive gc_refs. @@ -351,10 +351,10 @@ /* A traversal callback for move_unreachable. */ static int -visit_reachable(PyObject *op, PyGC_Head *reachable) +visit_reachable(PyObject **op, PyGC_Head *reachable) { - if (PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); + if (PyObject_IS_GC(*op)) { + PyGC_Head *gc = AS_GC(*op); const Py_ssize_t gc_refs = gc->gc.gc_refs; if (gc_refs == 0) { @@ -495,11 +495,11 @@ /* A traversal callback for move_finalizer_reachable. */ static int -visit_move(PyObject *op, PyGC_Head *tolist) +visit_move(PyObject **op, PyGC_Head *tolist) { - if (PyObject_IS_GC(op)) { - if (IS_TENTATIVELY_UNREACHABLE(op)) { - PyGC_Head *gc = AS_GC(op); + if (PyObject_IS_GC(*op)) { + if (IS_TENTATIVELY_UNREACHABLE(*op)) { + PyGC_Head *gc = AS_GC(*op); gc_list_move(gc, tolist); gc->gc.gc_refs = GC_REACHABLE; } @@ -1126,11 +1126,11 @@ } static int -referrersvisit(PyObject* obj, PyObject *objs) +referrersvisit(PyObject** obj, PyObject *objs) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(objs); i++) - if (PyTuple_GET_ITEM(objs, i) == obj) + if (PyTuple_GET_ITEM(objs, i) == *obj) return 1; return 0; } @@ -1176,9 +1176,9 @@ /* Append obj to list; return true if error (out of memory), false if OK. */ static int -referentsvisit(PyObject *obj, PyObject *list) +referentsvisit(PyObject **obj, PyObject *list) { - return PyList_Append(list, obj) < 0; + return PyList_Append(list, *obj) < 0; } PyDoc_STRVAR(gc_get_referents__doc__, @@ -1211,6 +1211,102 @@ return result; } +static int +remapvisit(PyObject** obj, PyObject *mapping) +{ + if (*obj && PyMapping_HasKey(mapping, *obj)) + { + /* The new reference returned by PyObject_GetItems will be + claimed by (*obj). The old reference is decremented only + after the new value has been assigned */ + PyObject * old_value = *obj; + PyObject * value = PyObject_GetItem(mapping, *obj); + (*obj) = value; + Py_DECREF(old_value); + } + return 0; +} + +static int +gc_remap_for(PyObject *objs, PyGC_Head *list, + PyObject *mapping, PyObject *ignore) +{ + PyGC_Head *gc; + PyObject *obj; + traverseproc traverse; + for (gc = list->gc.gc_next; gc != list; gc = gc->gc.gc_next) { + if (gc->gc.gc_refs != GC_REACHABLE) + continue; + + obj = FROM_GC(gc); + + if (obj == objs || obj == mapping) + continue; + + if (ignore) { + int result; + if (obj == ignore) + continue; + result = PySequence_Contains(ignore, obj); + if (result == 1) { + continue; + } else if (result == -1) { + PyErr_Clear(); + } + } + + traverse = Py_TYPE(obj)->tp_traverse; + if (! traverse) + continue; + (void) traverse(obj, (visitproc)remapvisit, mapping); + } + return 1; +} + +PyDoc_STRVAR(gc_remap__doc__, +"remap(mapping, ignore=None)\n\ +Update references to heap objects using the supplied mapping.\n\ +If ignore is not None, referencs inside objects in ignore are skipped.\n"); + +static PyObject * +gc_remap(PyObject *self, PyObject *args, PyObject *kws) +{ + static char *keywords[] = {"mapping", "ignore", NULL}; + int i; + PyObject *mapping; + PyObject *ignore = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kws, "O|O", keywords, &mapping, &ignore)) { + return NULL; + } + if (!PyMapping_Check(mapping)) { + PyErr_SetString(PyExc_ValueError, "Argument must be a mapping"); + return NULL; + } + /* Typically it is not necessary to incref arguments, however remapping may + eliminate the last referenc to mapping even from the argument tuple. */ + Py_INCREF(mapping); + Py_XINCREF(ignore); + + for (i = 0; i < NUM_GENERATIONS; i++) { + if (!(gc_remap_for(args, GEN_HEAD(i), mapping, ignore))) { + Py_XDECREF(ignore); + Py_DECREF(mapping); + return NULL; + } + } + + /* It is possible that remapping changed the base classes or mro of a + type. Because it is difficult to be sure we conservatively clear + the entire type cache. */ + PyType_ClearCache(); + + Py_XDECREF(ignore); + Py_DECREF(mapping); + Py_INCREF(Py_None); + return Py_None; +} + +/* Append obj to list; return true if error (out of memory), false if OK. */ PyDoc_STRVAR(gc_get_objects__doc__, "get_objects() -> [...]\n" "\n" @@ -1271,7 +1367,8 @@ "get_objects() -- Return a list of all objects tracked by the collector.\n" "is_tracked() -- Returns true if a given object is tracked.\n" "get_referrers() -- Return the list of objects that refer to an object.\n" -"get_referents() -- Return the list of objects that an object refers to.\n"); +"get_referents() -- Return the list of objects that an object refers to.\n" +"remap() -- Update references to heap objects using the supplied mapping.\n"); static PyMethodDef GcMethods[] = { {"enable", gc_enable, METH_NOARGS, gc_enable__doc__}, @@ -1290,6 +1387,8 @@ gc_get_referrers__doc__}, {"get_referents", gc_get_referents, METH_VARARGS, gc_get_referents__doc__}, + {"remap", (PyCFunction)gc_remap, METH_VARARGS | METH_KEYWORDS, + gc_remap__doc__}, {NULL, NULL} /* Sentinel */ }; Index: Modules/itertoolsmodule.c =================================================================== --- Modules/itertoolsmodule.c (revision 85840) +++ Modules/itertoolsmodule.c (working copy) @@ -489,7 +489,7 @@ static int tee_traverse(teeobject *to, visitproc visit, void *arg) { - Py_VISIT((PyObject *)to->dataobj); + Py_VISIT(to->dataobj); return 0; }