Index: Lib/pickle.py =================================================================== --- Lib/pickle.py (revision 63457) +++ Lib/pickle.py (working copy) @@ -33,6 +33,7 @@ import sys import struct import re +import collections __all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler", "Unpickler", "dump", "dumps", "load", "loads"] @@ -221,7 +222,20 @@ """Write a pickled representation of obj to the open file.""" if self.proto >= 2: self.write(PROTO + chr(self.proto)) - self.save(obj) + + # By faking recursion using generators, pickle is no longer dependent + # on python's recursion limit. This means that hugely recursive data + # structures can be pickled without a problem! It's also still just + # about as fast as it was, too. + callstack = collections.deque([self.save(obj)]) + while callstack: + try: + result = callstack[-1].next() + except StopIteration: + callstack.pop() + else: + if result is not None: + callstack.append(result) self.write(STOP) def memoize(self, obj): @@ -270,7 +284,7 @@ # Check for persistent id (defined by a subclass) pid = self.persistent_id(obj) if pid: - self.save_pers(pid) + yield self.save_pers(pid) return # Check the memo @@ -283,7 +297,7 @@ t = type(obj) f = self.dispatch.get(t) if f: - f(self, obj) # Call unbound method with explicit self + yield f(self, obj) # Call unbound method with explicit self return # Check for a class with a custom metaclass; treat as regular class @@ -328,7 +342,7 @@ "two to five elements" % reduce) # Save the reduce() output and finally memoize the object - self.save_reduce(obj=obj, *rv) + yield self.save_reduce(obj=obj, *rv) def persistent_id(self, obj): # This exists so a subclass can override it @@ -337,7 +351,7 @@ def save_pers(self, pid): # Save a persistent id reference if self.bin: - self.save(pid) + yield self.save(pid) self.write(BINPERSID) else: self.write(PERSID + str(pid) + '\n') @@ -393,12 +407,12 @@ raise PicklingError( "args[0] from __newobj__ args has the wrong class") args = args[1:] - save(cls) - save(args) + yield save(cls) + yield save(args) write(NEWOBJ) else: - save(func) - save(args) + yield save(func) + yield save(args) write(REDUCE) if obj is not None: @@ -410,13 +424,13 @@ # items and dict items (as (key, value) tuples), or None. if listitems is not None: - self._batch_appends(listitems) + yield self._batch_appends(listitems) if dictitems is not None: - self._batch_setitems(dictitems) + yield self._batch_setitems(dictitems) if state is not None: - save(state) + yield save(state) write(BUILD) # Methods below this point are dispatched through the dispatch table @@ -545,7 +559,7 @@ memo = self.memo if n <= 3 and proto >= 2: for element in obj: - save(element) + yield save(element) # Subtle. Same as in the big comment below. if id(obj) in memo: get = self.get(memo[id(obj)][0]) @@ -559,7 +573,7 @@ # has more than 3 elements. write(MARK) for element in obj: - save(element) + yield save(element) if id(obj) in memo: # Subtle. d was not in memo when we entered save_tuple(), so @@ -597,7 +611,7 @@ write(MARK + LIST) self.memoize(obj) - self._batch_appends(iter(obj)) + yield self._batch_appends(iter(obj)) dispatch[ListType] = save_list @@ -612,7 +626,7 @@ if not self.bin: for x in items: - save(x) + yield save(x) write(APPEND) return @@ -630,10 +644,10 @@ if n > 1: write(MARK) for x in tmp: - save(x) + yield save(x) write(APPENDS) elif n: - save(tmp[0]) + yield save(tmp[0]) write(APPEND) # else tmp is empty, and we're done @@ -646,7 +660,7 @@ write(MARK + DICT) self.memoize(obj) - self._batch_setitems(obj.iteritems()) + yield self._batch_setitems(obj.iteritems()) dispatch[DictionaryType] = save_dict if not PyStringMap is None: @@ -659,8 +673,8 @@ if not self.bin: for k, v in items: - save(k) - save(v) + yield save(k) + yield save(v) write(SETITEM) return @@ -677,13 +691,13 @@ if n > 1: write(MARK) for k, v in tmp: - save(k) - save(v) + yield save(k) + yield save(v) write(SETITEMS) elif n: k, v = tmp[0] - save(k) - save(v) + yield save(k) + yield save(v) write(SETITEM) # else tmp is empty, and we're done @@ -704,13 +718,13 @@ write(MARK) if self.bin: - save(cls) + yield save(cls) for arg in args: - save(arg) + yield save(arg) write(OBJ) else: for arg in args: - save(arg) + yield save(arg) write(INST + cls.__module__ + '\n' + cls.__name__ + '\n') self.memoize(obj) @@ -722,7 +736,7 @@ else: stuff = getstate() _keep_alive(stuff, memo) - save(stuff) + yield save(stuff) write(BUILD) dispatch[InstanceType] = save_inst