diff -r 77f02ab68a8a Lib/pickle.py --- a/Lib/pickle.py Mon Jul 19 20:17:19 2010 +0200 +++ b/Lib/pickle.py Mon Jul 19 15:54:01 2010 -0700 @@ -234,7 +234,22 @@ "%s.__init__()" % (self.__class__.__name__,)) if self.proto >= 2: self.write(PROTO + bytes([self.proto])) - self.save(obj) + + # By faking recursion using generators, pickle is no longer dependent + # on python's recursion limit. This means that hugely recursive data + # structures can be pickled without a problem! It's also still just + # about as fast as it was for simple structures, albeit slower for + # large structures. + callstack = [self.save(obj)] + while callstack: + try: + result = next(callstack[-1]) + except StopIteration: + callstack.pop() + else: + if result is not None: + callstack.append(result) + self.write(STOP) def memoize(self, obj): @@ -283,7 +298,7 @@ # Check for persistent id (defined by a subclass) pid = self.persistent_id(obj) if pid is not None and save_persistent_id: - self.save_pers(pid) + yield self.save_pers(pid) return # Check the memo @@ -296,7 +311,7 @@ t = type(obj) f = self.dispatch.get(t) if f: - f(self, obj) # Call unbound method with explicit self + yield f(self, obj) # Call unbound method with explicit self return # Check for a class with a custom metaclass; treat as regular class @@ -305,7 +320,7 @@ except TypeError: # t is not a class (old Boost; see SF #502085) issc = 0 if issc: - self.save_global(obj) + yield self.save_global(obj) return # Check copyreg.dispatch_table @@ -327,7 +342,7 @@ # Check for string returned by reduce(), meaning "save as global" if isinstance(rv, str): - self.save_global(obj, rv) + yield self.save_global(obj, rv) return # Assert that reduce() returned a tuple @@ -341,7 +356,7 @@ "two to five elements" % reduce) # Save the reduce() output and finally memoize the object - self.save_reduce(obj=obj, *rv) + yield self.save_reduce(obj=obj, *rv) def persistent_id(self, obj): # This exists so a subclass can override it @@ -350,7 +365,7 @@ def save_pers(self, pid): # Save a persistent id reference if self.bin: - self.save(pid, save_persistent_id=False) + yield self.save(pid, save_persistent_id=False) self.write(BINPERSID) else: self.write(PERSID + str(pid).encode("ascii") + b'\n') @@ -406,12 +421,12 @@ raise PicklingError( "args[0] from __newobj__ args has the wrong class") args = args[1:] - save(cls) - save(args) + yield save(cls) + yield save(args) write(NEWOBJ) else: - save(func) - save(args) + yield save(func) + yield save(args) write(REDUCE) if obj is not None: @@ -423,13 +438,13 @@ # items and dict items (as (key, value) tuples), or None. if listitems is not None: - self._batch_appends(listitems) + yield self._batch_appends(listitems) if dictitems is not None: - self._batch_setitems(dictitems) + yield self._batch_setitems(dictitems) if state is not None: - save(state) + yield save(state) write(BUILD) # Methods below this point are dispatched through the dispatch table @@ -487,7 +502,7 @@ def save_bytes(self, obj, pack=struct.pack): if self.proto < 3: - self.save_reduce(bytes, (list(obj),), obj=obj) + yield self.save_reduce(bytes, (list(obj),), obj=obj) return n = len(obj) if n < 256: @@ -526,7 +541,7 @@ memo = self.memo if n <= 3 and proto >= 2: for element in obj: - save(element) + yield save(element) # Subtle. Same as in the big comment below. if id(obj) in memo: get = self.get(memo[id(obj)][0]) @@ -540,7 +555,7 @@ # has more than 3 elements. write(MARK) for element in obj: - save(element) + yield save(element) if id(obj) in memo: # Subtle. d was not in memo when we entered save_tuple(), so @@ -572,7 +587,7 @@ write(MARK + LIST) self.memoize(obj) - self._batch_appends(obj) + yield self._batch_appends(obj) dispatch[list] = save_list @@ -585,7 +600,7 @@ if not self.bin: for x in items: - save(x) + yield save(x) write(APPEND) return @@ -604,10 +619,10 @@ if n > 1: write(MARK) for x in tmp: - save(x) + yield save(x) write(APPENDS) elif n: - save(tmp[0]) + yield save(tmp[0]) write(APPEND) # else tmp is empty, and we're done @@ -620,7 +635,7 @@ write(MARK + DICT) self.memoize(obj) - self._batch_setitems(obj.items()) + yield self._batch_setitems(obj.items()) dispatch[dict] = save_dict if PyStringMap is not None: @@ -633,8 +648,8 @@ if not self.bin: for k, v in items: - save(k) - save(v) + yield save(k) + yield save(v) write(SETITEM) return @@ -652,13 +667,13 @@ if n > 1: write(MARK) for k, v in tmp: - save(k) - save(v) + yield save(k) + yield save(v) write(SETITEMS) elif n: k, v = tmp[0] - save(k) - save(v) + yield save(k) + yield save(v) write(SETITEM) # else tmp is empty, and we're done diff -r 77f02ab68a8a Lib/test/pickletester.py --- a/Lib/test/pickletester.py Mon Jul 19 20:17:19 2010 +0200 +++ b/Lib/test/pickletester.py Mon Jul 19 15:54:01 2010 -0700 @@ -1069,6 +1069,21 @@ self.assertEqual(dumped, DATA6) + def test_recursion_depth(self): + root_list = prev_list = [] + for x in range(2000): + new_list = [x] + prev_list.append(new_list) + prev_list = new_list + # We're just testing if the data can be pickled. assertEquals doesn't + # really work because we'd exceed maximum recursion depth in cmp. + try: + for proto in protocols: + self.dumps(root_list, proto) + except RuntimeError as e: + self.fail('Got a RuntimeError when pickling: %s' % e) + + # Test classes for reduce_ex class REX_one(object): diff -r 77f02ab68a8a Lib/test/test_pickle.py --- a/Lib/test/test_pickle.py Mon Jul 19 20:17:19 2010 +0200 +++ b/Lib/test/test_pickle.py Mon Jul 19 15:54:01 2010 -0700 @@ -71,6 +71,10 @@ class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler + + def test_recursion_depth(self): + # Currently unimplemented in the C version. + pass class CPersPicklerTests(PyPersPicklerTests): pickler = _pickle.Pickler @@ -79,6 +83,10 @@ class CDumpPickle_LoadPickle(PyPicklerTests): pickler = _pickle.Pickler unpickler = pickle._Unpickler + + def test_recursion_depth(self): + # Currently unimplemented in the C version. + pass class DumpPickle_CLoadPickle(PyPicklerTests): pickler = pickle._Pickler