# HG changeset patch # User Antoine Pitrou # Date 1229548766 -3600 diff -r 8c7f13fb1f41 -r db35f879d3b8 Include/dictobject.h --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -86,6 +86,7 @@ PyDictEntry *ma_table; PyDictEntry *(*ma_lookup)(PyDictObject *mp, PyObject *key, long hash); PyDictEntry ma_smalltable[PyDict_MINSIZE]; + char ma_istracked; }; PyAPI_DATA(PyTypeObject) PyDict_Type; diff -r 8c7f13fb1f41 -r db35f879d3b8 Include/objimpl.h --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -285,11 +285,17 @@ g->gc.gc_next = NULL; \ } while (0); -/* True if the object is tracked by the GC. This can be useful to implement - some optimizations. */ +/* True if the object is currently tracked by the GC. */ #define _PyObject_GC_IS_TRACKED(o) \ ((_Py_AS_GC(o))->gc.gc_refs != _PyGC_REFS_UNTRACKED) +/* True if the object may be tracked by the GC in the future, or already is. + This can be useful to implement some optimizations. */ +#define _PyObject_GC_MAY_BE_TRACKED(obj) \ + (PyObject_IS_GC(obj) && \ + (!PyTuple_CheckExact(obj) || _PyObject_GC_IS_TRACKED(obj))) + + PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t); PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *); diff -r 8c7f13fb1f41 -r db35f879d3b8 Lib/test/test_dict.py --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1,7 +1,7 @@ import unittest from test import test_support -import UserDict, random, string +import UserDict, random, string, gc class DictTest(unittest.TestCase): @@ -554,6 +554,94 @@ pass d = {} + def _not_tracked(self, t): + self.assertFalse(gc.is_tracked(t), t) + + def _tracked(self, t): + self.assertTrue(gc.is_tracked(t), t) + + def test_track_literals(self): + # Test GC-optimization of dict literals + x, y, z, w = 1.5, "a", (1, None), [] + + self._not_tracked({}) + self._not_tracked({x:(), y:x, z:1}) + self._not_tracked({1: "a", "b": 2}) + self._not_tracked({1: 2, (None, True, False, ()): int}) + self._not_tracked({1: object()}) + + # Dicts with mutable elements are always tracked, even if those + # elements are not tracked right now. + self._tracked({1: []}) + self._tracked({1: ([],)}) + self._tracked({1: {}}) + self._tracked({1: set()}) + + def test_track_dynamic(self): + # Test GC-optimization of dynamically-created dicts + class MyObject(object): + pass + x, y, z, w, o = 1.5, "a", (1, object()), [], MyObject() + + d = dict() + self._not_tracked(d) + d[1] = "a" + self._not_tracked(d) + d[y] = 2 + self._not_tracked(d) + d[z] = 3 + self._not_tracked(d) + self._not_tracked(d.copy()) + d[4] = w + self._tracked(d) + self._tracked(d.copy()) + + # dd isn't tracked right now, but it may mutate and therefore d + # which contains it must be tracked. + d = dict() + dd = dict() + d[1] = dd + self._not_tracked(dd) + self._tracked(d) + dd[1] = d + self._tracked(dd) + + d = dict.fromkeys([x, y, z]) + self._not_tracked(d) + dd = dict() + dd.update(d) + self._not_tracked(dd) + d = dict.fromkeys([x, y, z, o]) + self._tracked(d) + dd = dict() + dd.update(d) + self._tracked(dd) + + d = dict(x=x, y=y, z=z) + self._not_tracked(d) + d = dict(x=x, y=y, z=z, w=w) + self._tracked(d) + d = dict() + d.update(x=x, y=y, z=z) + self._not_tracked(d) + d.update(w=w) + self._tracked(d) + + d = dict([(x, y), (z, 1)]) + self._not_tracked(d) + d = dict([(x, y), (z, w)]) + self._tracked(d) + d = dict() + d.update([(x, y), (z, 1)]) + self._not_tracked(d) + d.update([(x, y), (z, w)]) + self._tracked(d) + + def test_track_subtypes(self): + # Dict subtypes are always tracked + class MyDict(dict): + pass + self._tracked(MyDict()) from test import mapping_tests diff -r 8c7f13fb1f41 -r db35f879d3b8 Lib/test/test_gc.py --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -444,7 +444,6 @@ self.assertTrue(gc.is_tracked(NewStyle)) self.assertTrue(gc.is_tracked(NewStyle())) self.assertTrue(gc.is_tracked([])) - self.assertTrue(gc.is_tracked({})) self.assertTrue(gc.is_tracked(set())) def test_bug1055820b(self): diff -r 8c7f13fb1f41 -r db35f879d3b8 Lib/test/test_tuple.py --- a/Lib/test/test_tuple.py +++ b/Lib/test/test_tuple.py @@ -102,6 +102,8 @@ self._not_tracked((object(),)) self._not_tracked(((1, x), y, (2, 3))) + # Tuples with mutable elements are always tracked, even if those + # elements are not tracked right now. self._tracked(([],)) self._tracked(([1],)) self._tracked(({},)) @@ -120,7 +122,7 @@ check(tp(set([1, x, y]))) check(tp(tuple([obj]) for obj in [1, x, y])) check(tuple(tp([obj]) for obj in [1, x, y])) - + self._tracked(tp([z])) self._tracked(tp([[x, y]])) self._tracked(tp([{x: y}])) diff -r 8c7f13fb1f41 -r db35f879d3b8 Objects/abstract.c --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2151,8 +2151,7 @@ goto Fail; break; } - nb_tracked += (PyObject_IS_GC(item) && - _PyObject_GC_IS_TRACKED(item)); + nb_tracked += _PyObject_GC_MAY_BE_TRACKED(item); if (j >= n) { Py_ssize_t oldn = n; /* The over-allocation strategy can grow a bit faster diff -r 8c7f13fb1f41 -r db35f879d3b8 Objects/dictobject.c --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -180,6 +180,26 @@ } #endif +/* Debug statistic to count GC tracking of dicts */ +#undef SHOW_TRACK_COUNT +#ifdef SHOW_TRACK_COUNT +static Py_ssize_t count_untracked = 0; +static Py_ssize_t count_tracked = 0; + +static void +show_track(void) +{ + fprintf(stderr, "Dict creations: %" PY_FORMAT_SIZE_T "d\n", + count_tracked + count_untracked); + fprintf(stderr, "Dicts tracked by the GC: %" PY_FORMAT_SIZE_T + "d\n", count_tracked); + fprintf(stderr, "%.2f%% tracking rate\n\n", + (100.0*count_tracked/(count_untracked+count_tracked))); +} +#endif + + + /* Initialization macros. There are two ways to create a dict: PyDict_New() is the main C API function, and the tp_new slot maps to dict_new(). In the latter case we @@ -233,6 +253,9 @@ #ifdef SHOW_ALLOC_COUNT Py_AtExit(show_alloc); #endif +#ifdef SHOW_TRACK_COUNT + Py_AtExit(show_track); +#endif } if (numfree) { mp = free_list[--numfree]; @@ -262,10 +285,13 @@ #endif } mp->ma_lookup = lookdict_string; + mp->ma_istracked = 0; +#ifdef SHOW_TRACK_COUNT + count_untracked++; +#endif #ifdef SHOW_CONVERSION_COUNTS ++created; #endif - _PyObject_GC_TRACK(mp); return (PyObject *)mp; } @@ -433,6 +459,25 @@ return 0; } +#ifdef SHOW_TRACK_COUNT +#define INCREASE_TRACK_COUNT \ + (count_tracked++, count_untracked--); +#else +#define INCREASE_TRACK_COUNT +#endif + +#define MAINTAIN_TRACKING(mp, key, value) \ + do { \ + if (!mp->ma_istracked) { \ + if (_PyObject_GC_MAY_BE_TRACKED(key) || \ + _PyObject_GC_MAY_BE_TRACKED(value)) { \ + mp->ma_istracked = 1; \ + _PyObject_GC_TRACK(mp); \ + INCREASE_TRACK_COUNT \ + } \ + } \ + } while(0) + /* Internal routine to insert a new item into the table. Used both by the internal resize routine and by the public insert routine. @@ -453,6 +498,7 @@ Py_DECREF(value); return -1; } + MAINTAIN_TRACKING(mp, key, value); if (ep->me_value != NULL) { old_value = ep->me_value; ep->me_value = value; @@ -492,6 +538,7 @@ PyDictEntry *ep0 = mp->ma_table; register PyDictEntry *ep; + MAINTAIN_TRACKING(mp, key, value); i = hash & mask; ep = &ep0[i]; for (perturb = hash; ep->me_key != NULL; perturb >>= PERTURB_SHIFT) { @@ -2202,9 +2249,19 @@ assert(d->ma_table == NULL && d->ma_fill == 0 && d->ma_used == 0); INIT_NONZERO_DICT_SLOTS(d); d->ma_lookup = lookdict_string; + /* The object has been implicitely tracked by tp_alloc */ + d->ma_istracked = (type != &PyDict_Type); + if (!d->ma_istracked) + _PyObject_GC_UNTRACK(d); #ifdef SHOW_CONVERSION_COUNTS ++created; #endif +#ifdef SHOW_TRACK_COUNT + if (d->ma_istracked) + count_tracked++; + else + count_untracked++; +#endif } return self; } diff -r 8c7f13fb1f41 -r db35f879d3b8 Objects/tupleobject.c --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -147,7 +147,7 @@ return; for (i = 0; i < n; i++) { PyObject *elt = op->ob_item[i]; - if (PyObject_IS_GC(elt) && _PyObject_GC_IS_TRACKED(elt)) + if (_PyObject_GC_MAY_BE_TRACKED(elt)) return; } _PyObject_GC_UNTRACK(_op); @@ -171,8 +171,7 @@ o = va_arg(vargs, PyObject *); Py_INCREF(o); items[i] = o; - nb_tracked += (PyObject_IS_GC(o) - && _PyObject_GC_IS_TRACKED(o)); + nb_tracked += _PyObject_GC_MAY_BE_TRACKED(o); } if (n && !nb_tracked) _PyObject_GC_UNTRACK(result); diff -r 8c7f13fb1f41 -r db35f879d3b8 Python/ceval.c --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2021,8 +2021,7 @@ w = POP(); PyTuple_SET_ITEM(x, oparg, w); nb_tracked += - (PyObject_IS_GC(w) && - _PyObject_GC_IS_TRACKED(w)); + _PyObject_GC_MAY_BE_TRACKED(w); } if (!nb_tracked) _PyObject_GC_UNTRACK(x); diff -r 8c7f13fb1f41 -r db35f879d3b8 Python/modsupport.c --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -270,8 +270,7 @@ Py_INCREF(Py_None); w = Py_None; } - else nb_tracked += (PyObject_IS_GC(w) - && _PyObject_GC_IS_TRACKED(w)); + else nb_tracked += _PyObject_GC_MAY_BE_TRACKED(w); PyTuple_SET_ITEM(v, i, w); } if (itemfailed) {