Property changes on: . ___________________________________________________________________ Name: svnmerge-integrated - /python/trunk:1-61437,61439-61441,61443-61453,61455-61474,61476-61477,61479-61485,61487-61488,61490,61493-61528,61530-61563,61565-61569,61571-61576,61578-61584,61586,61588-61589,61591-61778,61780-61809,61811-61866,61868-61870,61872-61874,61876-61877,61883-61903,61905-61935,61938-61939,61941-62004,62007-62014,62016-62018,62021-62022,62024,62027,62029,62031-62066,62068-62074,62076-62094,62096,62098,62100-62102,62104-62122,62124-62125,62127-62137,62139-62142,62144-62194,62196-62222,62224-62308,62311-62397,62399-62430,62432-62510,62512-62516,62519-62535,62537-62598,62600-62665,62667-62683,62685-62719,62721,62723-62792,62794-62796,62798-62860,62862-62864,62866-62878,62880-62883,62885-62887,62889-62899,62901-62911,62913-62916,62918-62919,62921-62922,62924-62942,62944-62959,62961,62963-62967,62969-62970,62972-62973,62975-62976,62978-62982,62984,62987-63003,63005-63006,63009-63012,63014-63017,63019-63020,63022-63024,63026-63029,63031-63045,63047-63054,63056-63079,63081-63085,63087-63097,63099,63101-63104,63106-63107,63109-63113,63115-63128,63130-63133,63135-63144,63146-63148,63151-63152,63155-63165,63167-63176,63181-63186,63188-63190,63192-63194,63196-63206,63208-63209,63211-63212,63214-63217,63219-63224,63226-63227,63229-63232,63234-63235,63237-63239,63241,63243-63248,63250-63254,63256-63259,63261,63263-63264,63266-63267,63269-63270,63272-63273,63275-63276,63278,63280-63281,63283-63284,63286-63287,63289-63290,63292-63293,63295-63296,63298-63299,63301-63302,63304-63305,63307,63309-63314,63316-63322,63324-63325,63327-63335,63337-63342,63344-63346,63348-63353,63355-63357,63359-63373,63375,63377-63380,63383,63385-63391,63393-63402,63404-63407,63411-63412,63416-63420,63423-63424,63427,63429,63431-63433,63435-63439,63441-63457,63459-63468,63470-63480,63482-63484,63492,63496-63497,63499-63512,63514-63527,63529-63536,63538-63544,63546,63548,63550-63567,63569-63585,63590-63596,63602,63608-63610,63612-63616,63619-63629,63631-63648,63650-63659,63661-63664,63666-63671,63673-63674,63676,63678,63680-63684,63687-63690,63692-63703,63705-63713,63715-63717,63720,63722-63724,63726-63735,63737,63739-63741,63743-63744,63746-63766,63768-63775,63777-63798,63801-63813,63815-63845,63847,63850-63860,63862,63864-63872,63874-63898,63900-63909,63911-63913,63915-63931,63933-63941,63944,63946-63960,63964,63966,63968-63981,63983-64015,64017,64020-64027,64029-64030,64032-64039,64041-64043,64046-64047,64049-64056,64059-64061,64063-64067,64070-64079,64081-64085,64087-64089,64091-64092,64094-64096,64098-64104,64106-64127,64130-64132,64134-64141,64143-64168,64170-64205,64207-64247,64249,64251-64253,64255-64370,64372-64385,64387-64391,64393-64397,64399,64401-64412,64414-64423,64425-64428,64430,64432-64435,64440-64452,64454-64455,64457-64461,64463-64470,64472-64488,64490-64497,64499-64507,64509-64510,64512-64515,64517-64519,64521-64526,64528-64534,64536-64545,64548-64553,64556-64571,64573-64577,64579,64581-64594,64596-64632,64634,64636-64648,64650-64672,64674-64701,64703-64761,64763-64790,64792-64853,64856-64857,64860,64862-64864,64867-64909,64911-64912,64914-64921,64923-64929,64931-64952,64954-64957,64959-64961,64963-64974,64976-64977,64979,64981-64985,64987-65004 + /python/trunk:1-61437,61439-61441,61443-61453,61455-61474,61476-61477,61479-61485,61487-61488,61490,61493-61528,61530-61563,61565-61569,61571-61576,61578-61584,61586,61588-61589,61591-61778,61780-61809,61811-61866,61868-61870,61872-61874,61876-61877,61883-61903,61905-61935,61938-61939,61941-62004,62007-62014,62016-62018,62021-62022,62024,62027,62029,62031-62066,62068-62074,62076-62094,62096,62098,62100-62102,62104-62122,62124-62125,62127-62137,62139-62142,62144-62194,62196-62222,62224-62308,62311-62397,62399-62430,62432-62510,62512-62516,62519-62535,62537-62598,62600-62665,62667-62683,62685-62719,62721,62723-62792,62794-62796,62798-62860,62862-62864,62866-62878,62880-62883,62885-62887,62889-62899,62901-62911,62913-62916,62918-62919,62921-62922,62924-62942,62944-62959,62961,62963-62967,62969-62970,62972-62973,62975-62976,62978-62982,62984,62987-63003,63005-63006,63009-63012,63014-63017,63019-63020,63022-63024,63026-63029,63031-63045,63047-63054,63056-63079,63081-63085,63087-63097,63099,63101-63104,63106-63107,63109-63113,63115-63128,63130-63133,63135-63144,63146-63148,63151-63152,63155-63165,63167-63176,63181-63186,63188-63190,63192-63194,63196-63206,63208-63209,63211-63212,63214-63217,63219-63224,63226-63227,63229-63232,63234-63235,63237-63239,63241,63243-63248,63250-63254,63256-63259,63261,63263-63264,63266-63267,63269-63270,63272-63273,63275-63276,63278,63280-63281,63283-63284,63286-63287,63289-63290,63292-63293,63295-63296,63298-63299,63301-63302,63304-63305,63307,63309-63314,63316-63322,63324-63325,63327-63335,63337-63342,63344-63346,63348-63353,63355-63357,63359-63373,63375,63377-63380,63383,63385-63391,63393-63402,63404-63407,63411-63412,63416-63420,63423-63424,63427,63429,63431-63433,63435-63439,63441-63457,63459-63468,63470-63480,63482-63484,63492,63496-63497,63499-63512,63514-63527,63529-63536,63538-63544,63546,63548,63550-63567,63569-63585,63590-63596,63602,63608-63610,63612-63616,63619-63629,63631-63648,63650-63659,63661-63664,63666-63671,63673-63674,63676,63678,63680-63684,63687-63690,63692-63703,63705-63713,63715-63717,63720,63722-63724,63726-63735,63737,63739-63741,63743-63744,63746-63766,63768-63775,63777-63798,63801-63813,63815-63845,63847,63850-63860,63862,63864-63872,63874-63898,63900-63909,63911-63913,63915-63931,63933-63941,63944,63946-63960,63964,63966,63968-63981,63983-64015,64017,64020-64027,64029-64030,64032-64039,64041-64043,64046-64047,64049-64056,64059-64061,64063-64067,64070-64079,64081-64085,64087-64089,64091-64092,64094-64096,64098-64104,64106-64127,64130-64132,64134-64141,64143-64168,64170-64205,64207-64247,64249,64251-64253,64255-64370,64372-64385,64387-64391,64393-64397,64399,64401-64412,64414-64423,64425-64428,64430,64432-64435,64440-64452,64454-64455,64457-64461,64463-64470,64472-64488,64490-64497,64499-64507,64509-64510,64512-64515,64517-64519,64521-64526,64528-64534,64536-64545,64548-64553,64556-64571,64573-64577,64579,64581-64594,64596-64632,64634,64636-64648,64650-64672,64674-64701,64703-64761,64763-64790,64792-64853,64856-64857,64860,64862-64864,64867-64909,64911-64912,64914-64921,64923-64929,64931-64952,64954-64957,64959-64961,64963-64974,64976-64977,64979,64981-64985,64987-65004,65016 Index: Misc/NEWS =================================================================== --- Misc/NEWS (revision 65011) +++ Misc/NEWS (working copy) @@ -34,6 +34,11 @@ Library ------- +- Issue #3125: Remove copy_reg in multiprocessing and replace it with + ForkingPickler.register() to resolve conflict with ctypes. + +- Issue #3090: Fixed ARCHFLAGS parsing on OS/X + - Issue #3313: Fixed a crash when a failed dlopen() call does not set a valid dlerror() message. Index: Lib/multiprocessing/reduction.py =================================================================== --- Lib/multiprocessing/reduction.py (revision 65011) +++ Lib/multiprocessing/reduction.py (working copy) @@ -13,11 +13,10 @@ import sys import socket import threading -import copyreg import _multiprocessing from multiprocessing import current_process -from multiprocessing.forking import Popen, duplicate, close +from multiprocessing.forking import Popen, duplicate, close, ForkingPickler from multiprocessing.util import register_after_fork, debug, sub_debug from multiprocessing.connection import Client, Listener @@ -134,7 +133,7 @@ return new_handle # -# Register `_multiprocessing.Connection` with `copy_reg` +# Register `_multiprocessing.Connection` with `ForkingPickler` # def reduce_connection(conn): @@ -146,11 +145,10 @@ return _multiprocessing.Connection( handle, readable=readable, writable=writable ) +ForkingPickler.register(_multiprocessing.Connection, reduce_connection) -copyreg.pickle(_multiprocessing.Connection, reduce_connection) - # -# Register `socket.socket` with `copy_reg` +# Register `socket.socket` with `ForkingPickler` # def fromfd(fd, family, type_, proto=0): @@ -169,10 +167,10 @@ close(fd) return _sock -copyreg.pickle(socket.socket, reduce_socket) +ForkingPickler.register(socket.socket, reduce_socket) # -# Register `_multiprocessing.PipeConnection` with `copy_reg` +# Register `_multiprocessing.PipeConnection` with `ForkingPickler` # if sys.platform == 'win32': @@ -187,4 +185,4 @@ handle, readable=readable, writable=writable ) - copyreg.pickle(_multiprocessing.PipeConnection, reduce_pipe_connection) + ForkingPickler.register(_multiprocessing.PipeConnection, reduce_pipe_connection) Index: Lib/multiprocessing/managers.py =================================================================== --- Lib/multiprocessing/managers.py (revision 65011) +++ Lib/multiprocessing/managers.py (working copy) @@ -18,17 +18,16 @@ import weakref import threading import array -import copyreg import queue from traceback import format_exc from multiprocessing import Process, current_process, active_children, Pool, util, connection from multiprocessing.process import AuthenticationString -from multiprocessing.forking import exit, Popen, assert_spawning +from multiprocessing.forking import exit, Popen, assert_spawning, ForkingPickler from multiprocessing.util import Finalize, info try: - from cPickle import PicklingError + from pickle import PicklingError except ImportError: from pickle import PicklingError @@ -38,14 +37,14 @@ def reduce_array(a): return array.array, (a.typecode, a.tostring()) -copyreg.pickle(array.array, reduce_array) +ForkingPickler.register(array.array, reduce_array) view_types = [type(getattr({}, name)()) for name in ('items','keys','values')] if view_types[0] is not list: # only needed in Py3.0 def rebuild_as_list(obj): return list, (list(obj),) for view_type in view_types: - copyreg.pickle(view_type, rebuild_as_list) + ForkingPickler.register(view_type, rebuild_as_list) # # Type for identifying shared objects Index: Lib/multiprocessing/util.py =================================================================== --- Lib/multiprocessing/util.py (revision 65011) +++ Lib/multiprocessing/util.py (working copy) @@ -8,7 +8,6 @@ import itertools import weakref -import copyreg import atexit import threading # we want threading to install it's # cleanup function before multiprocessing does @@ -302,35 +301,3 @@ register_after_fork(self, lambda obj : obj.__dict__.clear()) def __reduce__(self): return type(self), () - -# -# Try making some callable types picklable -# - -def _reduce_method(m): - if m.__self__ is None: - return getattr, (m.__self__.__class__, m.__func__.__name__) - else: - return getattr, (m.__self__, m.__func__.__name__) -copyreg.pickle(type(Finalize.__init__), _reduce_method) - -def _reduce_method_descriptor(m): - return getattr, (m.__objclass__, m.__name__) -copyreg.pickle(type(list.append), _reduce_method_descriptor) -copyreg.pickle(type(int.__add__), _reduce_method_descriptor) - -def _reduce_builtin_function_or_method(m): - return getattr, (m.__self__, m.__name__) -copyreg.pickle(type(list().append), _reduce_builtin_function_or_method) -copyreg.pickle(type(int().__add__), _reduce_builtin_function_or_method) - -try: - from functools import partial -except ImportError: - pass -else: - def _reduce_partial(p): - return _rebuild_partial, (p.func, p.args, p.keywords or {}) - def _rebuild_partial(func, args, keywords): - return partial(func, *args, **keywords) - copyreg.pickle(partial, _reduce_partial) Index: Lib/multiprocessing/sharedctypes.py =================================================================== --- Lib/multiprocessing/sharedctypes.py (revision 65011) +++ Lib/multiprocessing/sharedctypes.py (working copy) @@ -9,10 +9,9 @@ import sys import ctypes import weakref -import copyreg from multiprocessing import heap, RLock -from multiprocessing.forking import assert_spawning +from multiprocessing.forking import assert_spawning, ForkingPickler __all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized'] @@ -124,8 +123,7 @@ def rebuild_ctype(type_, wrapper, length): if length is not None: type_ = type_ * length - if sys.platform == 'win32' and type_ not in copyreg.dispatch_table: - copyreg.pickle(type_, reduce_ctype) + ForkingPickler.register(type_, reduce_ctype) obj = type_.from_address(wrapper.get_address()) obj._wrapper = wrapper return obj Index: Lib/multiprocessing/connection.py =================================================================== --- Lib/multiprocessing/connection.py (revision 65011) +++ Lib/multiprocessing/connection.py (working copy) @@ -399,10 +399,10 @@ return self._loads(s) def _xml_dumps(obj): - return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf8') + return xmlrpc.client.dumps((obj,), None, None, None, 1).encode('utf8') def _xml_loads(s): - (obj,), method = xmlrpclib.loads(s.decode('utf8')) + (obj,), method = xmlrpc.client.loads(s.decode('utf8')) return obj class XmlListener(Listener): Index: Lib/multiprocessing/forking.py =================================================================== --- Lib/multiprocessing/forking.py (revision 65011) +++ Lib/multiprocessing/forking.py (working copy) @@ -12,7 +12,7 @@ from multiprocessing import util, process -__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close'] +__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close', 'ForkingPickler'] # # Check that the current thread is spawning a child process @@ -26,6 +26,49 @@ ) # +# Try making some callable types picklable +# + +from pickle import Pickler +class ForkingPickler(Pickler): + dispatch = Pickler.dispatch.copy() + + @classmethod + def register(cls, type, reduce): + def dispatcher(self, obj): + rv = reduce(obj) + self.save_reduce(obj=obj, *rv) + cls.dispatch[type] = dispatcher + +def _reduce_method(m): + if m.__self__ is None: + return getattr, (m.__self__.__class__, m.__func__.__name__) + else: + return getattr, (m.__self__, m.__func__.__name__) +ForkingPickler.register(type(ForkingPickler.save), _reduce_method) + +def _reduce_method_descriptor(m): + return getattr, (m.__objclass__, m.__name__) +ForkingPickler.register(type(list.append), _reduce_method_descriptor) +ForkingPickler.register(type(int.__add__), _reduce_method_descriptor) + +#def _reduce_builtin_function_or_method(m): +# return getattr, (m.__self__, m.__name__) +#ForkingPickler.register(type(list().append), _reduce_builtin_function_or_method) +#ForkingPickler.register(type(int().__add__), _reduce_builtin_function_or_method) + +try: + from functools import partial +except ImportError: + pass +else: + def _reduce_partial(p): + return _rebuild_partial, (p.func, p.args, p.keywords or {}) + def _rebuild_partial(func, args, keywords): + return partial(func, *args, **keywords) + ForkingPickler.register(partial, _reduce_partial) + +# # Unix # @@ -105,17 +148,19 @@ import _thread import msvcrt import _subprocess - import copyreg import time from ._multiprocessing import win32, Connection, PipeConnection from .util import Finalize - try: - from cPickle import dump, load, HIGHEST_PROTOCOL - except ImportError: - from pickle import dump, load, HIGHEST_PROTOCOL + #try: + # from cPickle import dump, load, HIGHEST_PROTOCOL + #except ImportError: + from pickle import load, HIGHEST_PROTOCOL + def dump(obj, file, protocol=None): + ForkingPickler(file, protocol).dump(obj) + # # # @@ -346,10 +391,9 @@ return type(conn), (Popen.duplicate_for_child(conn.fileno()), conn.readable, conn.writable) - copyreg.pickle(Connection, reduce_connection) - copyreg.pickle(PipeConnection, reduce_connection) + ForkingPickler.register(Connection, reduce_connection) + ForkingPickler.register(PipeConnection, reduce_connection) - # # Prepare current process #