Index: Lib/test/test_pickle.py =================================================================== --- Lib/test/test_pickle.py (revision 71401) +++ Lib/test/test_pickle.py (working copy) @@ -64,6 +64,10 @@ class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler + + def test_recursion_depth(self): + # Currently unimplemented in the C version. + pass class CPersPicklerTests(PyPersPicklerTests): pickler = _pickle.Pickler Index: Lib/test/pickletester.py =================================================================== --- Lib/test/pickletester.py (revision 71401) +++ Lib/test/pickletester.py (working copy) @@ -917,6 +917,21 @@ except (pickle.PickleError): pass + def test_recursion_depth(self): + root_list = prev_list = [] + for x in range(2000): + new_list = [x] + prev_list.append(new_list) + prev_list = new_list + # We're just testing if the data can be pickled. assertEquals doesn't + # really work because we'd exceed maximum recursion depth in cmp. + try: + for proto in protocols: + self.dumps(root_list, proto) + except RuntimeError as e: + self.fail('Got a RuntimeError when pickling: %s' % e) + + # Test classes for reduce_ex class REX_one(object): Index: Lib/pickle.py =================================================================== --- Lib/pickle.py (revision 71401) +++ Lib/pickle.py (working copy) @@ -229,7 +229,22 @@ "%s.__init__()" % (self.__class__.__name__,)) if self.proto >= 2: self.write(PROTO + bytes([self.proto])) - self.save(obj) + + # By faking recursion using generators, pickle is no longer dependent + # on python's recursion limit. This means that hugely recursive data + # structures can be pickled without a problem! It's also still just + # about as fast as it was for simple structures, albeit slower for + # large structures. + callstack = [self.save(obj)] + while callstack: + try: + result = next(callstack[-1]) + except StopIteration: + callstack.pop() + else: + if result is not None: + callstack.append(result) + self.write(STOP) def memoize(self, obj): @@ -278,7 +293,7 @@ # Check for persistent id (defined by a subclass) pid = self.persistent_id(obj) if pid is not None and save_persistent_id: - self.save_pers(pid) + yield self.save_pers(pid) return # Check the memo @@ -291,7 +306,7 @@ t = type(obj) f = self.dispatch.get(t) if f: - f(self, obj) # Call unbound method with explicit self + yield f(self, obj) # Call unbound method with explicit self return # Check for a class with a custom metaclass; treat as regular class @@ -300,7 +315,7 @@ except TypeError: # t is not a class (old Boost; see SF #502085) issc = 0 if issc: - self.save_global(obj) + yield self.save_global(obj) return # Check copyreg.dispatch_table @@ -322,7 +337,7 @@ # Check for string returned by reduce(), meaning "save as global" if isinstance(rv, str): - self.save_global(obj, rv) + yield self.save_global(obj, rv) return # Assert that reduce() returned a tuple @@ -336,7 +351,7 @@ "two to five elements" % reduce) # Save the reduce() output and finally memoize the object - self.save_reduce(obj=obj, *rv) + yield self.save_reduce(obj=obj, *rv) def persistent_id(self, obj): # This exists so a subclass can override it @@ -345,7 +360,7 @@ def save_pers(self, pid): # Save a persistent id reference if self.bin: - self.save(pid, save_persistent_id=False) + yield self.save(pid, save_persistent_id=False) self.write(BINPERSID) else: self.write(PERSID + str(pid).encode("ascii") + b'\n') @@ -401,12 +416,12 @@ raise PicklingError( "args[0] from __newobj__ args has the wrong class") args = args[1:] - save(cls) - save(args) + yield save(cls) + yield save(args) write(NEWOBJ) else: - save(func) - save(args) + yield save(func) + yield save(args) write(REDUCE) if obj is not None: @@ -418,13 +433,13 @@ # items and dict items (as (key, value) tuples), or None. if listitems is not None: - self._batch_appends(listitems) + yield self._batch_appends(listitems) if dictitems is not None: - self._batch_setitems(dictitems) + yield self._batch_setitems(dictitems) if state is not None: - save(state) + yield save(state) write(BUILD) # Methods below this point are dispatched through the dispatch table @@ -482,7 +497,7 @@ def save_bytes(self, obj, pack=struct.pack): if self.proto < 3: - self.save_reduce(bytes, (list(obj),), obj=obj) + yield self.save_reduce(bytes, (list(obj),), obj=obj) return n = len(obj) if n < 256: @@ -521,7 +536,7 @@ memo = self.memo if n <= 3 and proto >= 2: for element in obj: - save(element) + yield save(element) # Subtle. Same as in the big comment below. if id(obj) in memo: get = self.get(memo[id(obj)][0]) @@ -535,7 +550,7 @@ # has more than 3 elements. write(MARK) for element in obj: - save(element) + yield save(element) if id(obj) in memo: # Subtle. d was not in memo when we entered save_tuple(), so @@ -567,7 +582,7 @@ write(MARK + LIST) self.memoize(obj) - self._batch_appends(obj) + yield self._batch_appends(obj) dispatch[list] = save_list @@ -580,7 +595,7 @@ if not self.bin: for x in items: - save(x) + yield save(x) write(APPEND) return @@ -599,10 +614,10 @@ if n > 1: write(MARK) for x in tmp: - save(x) + yield save(x) write(APPENDS) elif n: - save(tmp[0]) + yield save(tmp[0]) write(APPEND) # else tmp is empty, and we're done @@ -615,7 +630,7 @@ write(MARK + DICT) self.memoize(obj) - self._batch_setitems(obj.items()) + yield self._batch_setitems(obj.items()) dispatch[dict] = save_dict if PyStringMap is not None: @@ -628,8 +643,8 @@ if not self.bin: for k, v in items: - save(k) - save(v) + yield save(k) + yield save(v) write(SETITEM) return @@ -647,13 +662,13 @@ if n > 1: write(MARK) for k, v in tmp: - save(k) - save(v) + yield save(k) + yield save(v) write(SETITEMS) elif n: k, v = tmp[0] - save(k) - save(v) + yield save(k) + yield save(v) write(SETITEM) # else tmp is empty, and we're done