I have 10 identical test machines running this this code ( operating systems are cloned ). I am not usning valgrind in these tests, it was causing various issues...
(gdb) info sharedlibrary
From To Syms Read Shared Object Library
0xb75a02b0 0xb76baa38 Yes /usr/lib/libpython2.7.so.1.0
0xb75695a0 0xb7575878 Yes /lib/libpthread.so.0
0xb7422060 0xb7529374 Yes /lib/libc.so.6
0xb7404bd0 0xb7405b28 Yes /lib/libdl.so.2
0xb73ffbc0 0xb7400498 Yes /lib/libutil.so.1
0xb73dd360 0xb73f7e58 Yes /lib/libm.so.6
0xb77288d0 0xb773f97f Yes /lib/ld-linux.so.2
0xb7721310 0xb77226d8 Yes /usr/lib/python2.7/lib-dynload/time.so
0xb728c130 0xb728e4b8 Yes /usr/lib/python2.7/site-packages/zope/interface/_zope_interface_coptimizations.so
0xb7284060 0xb7286cf8 Yes /usr/lib/python2.7/lib-dynload/strop.so
0xb72772a0 0xb727d648 Yes /usr/lib/python2.7/lib-dynload/_socket.so
0xb7271c50 0xb72729e8 Yes /usr/lib/python2.7/lib-dynload/_functools.so
0xb726a7d0 0xb726deb8 Yes /usr/lib/python2.7/lib-dynload/_ssl.so
0xb721ec00 0xb7253f28 Yes /usr/lib/libssl.so.1.0.0
0xb70f47c0 0xb71d3dd8 Yes /usr/lib/libcrypto.so.1.0.0
0xb70a3b20 0xb70af7b8 Yes /lib/libz.so.1
0xb7264130 0xb72653c8 Yes /usr/lib/python2.7/lib-dynload/cStringIO.so
0xb709aab0 0xb709e108 Yes /usr/lib/python2.7/lib-dynload/_struct.so
0xb70934c0 0xb70958a8 Yes /usr/lib/python2.7/lib-dynload/operator.so
0xb708b7e0 0xb708e868 Yes /usr/lib/python2.7/lib-dynload/_collections.so
0xb707db00 0xb7083dd8 Yes /usr/lib/python2.7/lib-dynload/itertools.so
0xb725f7d0 0xb725ff98 Yes /usr/lib/python2.7/lib-dynload/_bisect.so
0xb7077920 0xb7078ee8 Yes /usr/lib/python2.7/lib-dynload/_heapq.so
0xb7073970 0xb7074028 Yes /usr/lib/python2.7/lib-dynload/grp.so
0xb771d510 0xb771d6f8 Yes /usr/lib/python2.7/site-packages/twisted/python/_initgroups.so
0xb7062bf0 0xb706d3f8 Yes /usr/lib/python2.7/lib-dynload/datetime.so
0xb705ab90 0xb705d358 Yes /usr/lib/python2.7/lib-dynload/binascii.so
0xb7055a00 0xb7057018 Yes /usr/lib/python2.7/lib-dynload/fcntl.so
0xb704d9a0 0xb7051258 Yes /usr/lib/python2.7/lib-dynload/math.so
0xb7046e80 0xb7047d48 Yes /usr/lib/python2.7/lib-dynload/_hashlib.so
0xb7042d90 0xb7043d78 Yes /usr/lib/python2.7/lib-dynload/_random.so
0xb703c460 0xb703e548 Yes /usr/lib/python2.7/lib-dynload/select.so
0xb7036360 0xb7036e18 Yes /usr/lib/python2.7/lib-dynload/termios.so
0xb7032660 0xb7032968 Yes /usr/lib/python2.7/site-packages/twisted/internet/_sigchld.so
0xb6ce7930 0xb6ed3188 Yes /usr/lib/python2.7/site-packages/PySide/QtCore.so
0xb6c903b0 0xb6ca3b58 Yes /usr/lib/libpyside-python2.7.so.1.0
0xb7016230 0xb7023058 Yes /usr/lib/libshiboken-python2.7.so.1.0
0xb69ba370 0xb6b4c358 Yes /usr/lib/qt4/libQtCore.so.4
0xb68cd7e0 0xb693dac8 Yes /usr/lib/gcc/i686-pc-linux-gnu/4.5.3/libstdc++.so.6
0xb686a390 0xb6880a18 Yes /usr/lib/gcc/i686-pc-linux-gnu/4.5.3/libgcc_s.so.1
0xb6860e90 0xb68655c8 Yes /lib/librt.so.1
---Type <return> to continue, or q <return> to quit---
0xb5d81c30 0xb65d9cf8 Yes /usr/lib/python2.7/site-packages/PySide/QtGui.so
0xb54d9e50 0xb5ba8a38 Yes /usr/lib/qt4/libQtGui.so.4
0xb5394b80 0xb53b0a58 Yes /usr/lib/libpng15.so.15
0xb5315a80 0xb5377748 Yes /usr/lib/libfreetype.so.6
0xb5305910 0xb530a278 Yes /usr/lib/libSM.so.6
0xb52eee00 0xb52fddd8 Yes /usr/lib/libICE.so.6
0xb52e3740 0xb52e8be8 Yes /usr/lib/libXrender.so.1
0xb52db5c0 0xb52dfa48 Yes /usr/lib/libXrandr.so.2
0xb52b0f50 0xb52ca548 Yes /usr/lib/libfontconfig.so.1
0xb529f220 0xb52a8e98 Yes /usr/lib/libXext.so.6
0xb519b580 0xb5220388 Yes /usr/lib/libX11.so.6
0xb5173200 0xb5181a38 Yes /lib/libbz2.so.1
0xb5151580 0xb5167338 Yes /usr/lib/libexpat.so.1
0xb513e0c0 0xb514b988 Yes /usr/lib/libxcb.so.1
0xb5131b60 0xb51328b8 Yes /usr/lib/libXau.so.6
0xb512c0f0 0xb512dab8 Yes /usr/lib/libXdmcp.so.6
0xb50417e0 0xb50f0cf8 Yes /usr/lib/python2.7/site-packages/PySide/QtNetwork.so
0xb4f13ff0 0xb4fd6f18 Yes /usr/lib/qt4/libQtNetwork.so.4
0xb702d1e0 0xb702ebd8 Yes /usr/lib/python2.7/lib-dynload/_locale.so
0xb4ed2ab0 0xb4ee4fa8 Yes /usr/lib/python2.7/site-packages/psycopg2/_psycopg.so
0xb4ea7d90 0xb4ebe638 Yes (*) /usr/lib/libpq.so.5
0xb4e8fc90 0xb4e9da98 Yes /usr/lib/python2.7/lib-dynload/cPickle.so
0xb4e84c70 0xb4e89498 Yes /usr/lib/python2.7/lib-dynload/array.so
0xb4e69d90 0xb4e7cb08 Yes /usr/lib/python2.7/lib-dynload/_ctypes.so
0xb4e5f310 0xb4e62638 Yes /usr/lib/libffi.so.5
0xb4ec8100 0xb4ec9ab8 Yes /lib/libuuid.so.1
0xb6fb2200 0xb6fb4618 Yes /usr/lib/libXfixes.so
0xb6fa93e0 0xb6fae248 Yes /usr/lib/libXcursor.so.1
0xb6f9ad70 0xb6fa4808 Yes /usr/lib/libXi.so
0xb4e316d0 0xb4e56308 Yes /usr/lib/python2.7/site-packages/PIL/_imaging.so
0xb4ddfff0 0xb4e134c8 Yes /usr/lib/libjpeg.so.8
0xb6f79d00 0xb6f8d748 Yes /usr/lib/python2.7/lib-dynload/_io.so
0xb6f6fef0 0xb6f724d8 Yes /usr/lib/python2.7/lib-dynload/zlib.so
0xb4d4c190 0xb4db81e8 Yes /usr/lib/python2.7/site-packages/PySide/QtWebKit.so
0xb4049e70 0xb4ade738 Yes /usr/lib/qt4/libQtWebKit.so.4
0xb3e22850 0xb3e90648 Yes /usr/lib/libsqlite3.so.0
0xb3de9c20 0xb3e13f28 Yes /usr/lib/qt4/libphonon.so.4
0xb3dcd0d0 0xb3dd0548 Yes /usr/lib/python2.7/site-packages/alsaaudio.so
0xb3d302c0 0xb3db2648 Yes /usr/lib/libasound.so.2
0xb6fb9b60 0xb6fbcef8 Yes /usr/lib/qt4/plugins/imageformats/libqgif.so
0xb3509f30 0xb350cd88 Yes /usr/lib/qt4/plugins/imageformats/libqico.so
0xaebf0710 0xaebf3c28 Yes /usr/lib/qt4/plugins/imageformats/libqjpeg.so
0xade9ad30 0xadea6eb8 Yes (*) /usr/lib/libresolv.so
0xade8dfa0 0xade955c8 Yes /lib/libnss_files.so.2
0xade86da0 0xade89e18 Yes /lib/libnss_dns.so.2
---Type <return> to continue, or q <return> to quit---
0xad63b080 0xad63c6e8 Yes /usr/lib/qt4/plugins/imageformats/libqsvg.so
0xad601ac0 0xad633038 Yes /usr/lib/qt4/libQtSvg.so.4
0xad4a5610 0xad4ab728 Yes /usr/lib/python2.7/lib-dynload/_json.so
0xad499640 0xad49f578 Yes /usr/lib/python2.7/lib-dynload/pyexpat.so
0xad018350 0xad01a708 Yes /usr/lib/python2.7/lib-dynload/unicodedata.so
I am importing twisted, pyside, django, psycopg2, and datetuil
3 of the machines had a stall much like the last two backtraces, except now originating from my gc.collect() call:
#0 0xb769e9ff in update_refs (containers=0xb7709328) at Modules/gcmodule.c:290
#1 0xb769f55f in collect (generation=2) at Modules/gcmodule.c:873
#2 0xb769f98d in gc_collect (self=0x0, args=(), kws=0x0) at Modules/gcmodule.c:1067
#3 0xb75f0e91 in PyCFunction_Call (func=<built-in function collect>, arg=(), kw=0x0)
at Objects/methodobject.c:85
#4 0xb7666287 in call_function (pp_stack=0xbfa61960, oparg=0) at Python/ceval.c:4013
#5 0xb7662dff in PyEval_EvalFrameEx (f=
Frame 0xb0b5b9c, for file /usr/lib/python2.7/site-packages/qt4reactor.py, line 261, in doIteration (self=<QtReactor(waker=<_UnixWaker(i=13, fileno=<function at remote 0xa688994>, reactor=<...>, o=14) at remote 0xa684dac>, threadCallQueue=[], _cancellations=0, _justStopped=False, _newTimedCalls=[], _pendingTimedCalls=[<DelayedCall(resetter=<instancemethod at remote 0xb00016c>, seconds=<built-in function time>, args=(), canceller=<instancemethod at remote 0xad81e64>, delayed_time=0, kw={}, func=<LoopingCall(a=(...), interval=2, clock=<...>, _expectNextCallAt=<float at remote 0xb03ac1c>, f=<instancemethod at remote 0xa97234c>, deferred=<Deferred(_canceller=None, callbacks=[((<function at remote 0x9e59f44>, None, None), (<instancemethod at remote 0xa972374>, (...), {}))]) at remote 0xa9710cc>, running=True, kw={}, starttime=<float at remote 0x9ed5f3c>, call=<...>, _runAtStart=True) at remote 0xa96e76c>, time=<float at remote 0xac30db4>, cancelled=0, called=0) at remote 0xb03482c>, <DelayedCall(resetter=<instancem...(truncated), throwflag=0) at Python/ceval.c:2666
this is weird, but it seems like the type pointer is 0x0 (no type?):
(gdb) print {PyObject} ((PyObject *)(((PyGC_Head *)*gc)+1))
$1 = {ob_refcnt = 700, ob_type = 0x0}
gc gets stuck again in update_refs, and if you look at its members, gc.gc_next == gc.gc_prev == gc ....
(gdb) print {PyGC_Head} gc
$3 = {gc = {gc_next = 0xb77a3300, gc_prev = 0xb77a3300, gc_refs = 700},
dummy = 1.2675072995965765876244656259641832e-4721}
Two of the machines produced core files, first core:
(Incidentally, this machine, once reset was one of the ones that was counted in the previous list that locked up at update_refs)
Program terminated with signal 6, Aborted.
#0 0xb76fe424 in __kernel_vsyscall ()
(gdb) info thread
Id Target Id Frame
12 Thread 0xb0afab70 (LWP 2752) 0xb76fe424 in __kernel_vsyscall ()
11 Thread 0xb12fbb70 (LWP 2751) 0xb76fe424 in __kernel_vsyscall ()
10 Thread 0xaf2f7b70 (LWP 2755) 0xb76fe424 in __kernel_vsyscall ()
9 Thread 0xb22fdb70 (LWP 2749) 0xb76fe424 in __kernel_vsyscall ()
8 Thread 0xb2afeb70 (LWP 2748) 0xb76fe424 in __kernel_vsyscall ()
7 Thread 0xb1afcb70 (LWP 2750) 0xb76fe424 in __kernel_vsyscall ()
6 Thread 0xb32ffb70 (LWP 2747) 0xb76fe424 in __kernel_vsyscall ()
5 Thread 0xaeaeeb70 (LWP 2757) 0xb76fe424 in __kernel_vsyscall ()
4 Thread 0xb02f9b70 (LWP 2753) 0xb76fe424 in __kernel_vsyscall ()
3 Thread 0xb3ce6b70 (LWP 2740) 0xb76fe424 in __kernel_vsyscall ()
2 Thread 0xafaf8b70 (LWP 2754) 0xb7039cda in __i686.get_pc_thunk.bx ()
from /usr/lib/python2.7/lib-dynload/datetime.so
* 1 Thread 0xb73af6c0 (LWP 2597) 0xb76fe424 in __kernel_vsyscall ()
I'm not at all sure whats going on here... but the file for the service ends with this:
Fatal Python error: GC object already tracked
The other core file is different:
Program terminated with signal 11, Segmentation fault.
#0 0xb766f3ee in call_function (pp_stack=0xbfda38c0, oparg=0) at Python/ceval.c:4047
4047 Python/ceval.c: No such file or directory.
in Python/ceval.c
#0 0xb766f3ee in call_function (pp_stack=0xbfda38c0, oparg=0) at Python/ceval.c:4047
#1 0xb766bdff in PyEval_EvalFrameEx (f=<unknown at remote 0xa0bf31c>, throwflag=0)
at Python/ceval.c:2666
#2 0xb766f542 in fast_function (func=<function at remote 0x922733c>, pp_stack=0xbfda3ac0,
n=1, na=1, nk=0) at Python/ceval.c:4099
#3 0xb766f383 in call_function (pp_stack=0xbfda3ac0, oparg=0) at Python/ceval.c:4034
#4 0xb766bdff in PyEval_EvalFrameEx (f=<unknown at remote 0xa0a69ac>, throwflag=0)
at Python/ceval.c:2666
#5 0xb766f542 in fast_function (func=<function at remote 0x92272cc>, pp_stack=0xbfda3cc0,
n=2, na=2, nk=0) at Python/ceval.c:4099
#6 0xb766f383 in call_function (pp_stack=0xbfda3cc0, oparg=1) at Python/ceval.c:4034
#7 0xb766bdff in PyEval_EvalFrameEx (f=<unknown at remote 0xa0c27a4>, throwflag=0)
at Python/ceval.c:2666
#8 0xb766d7ad in PyEval_EvalCodeEx (co=0x9212cc8, globals=
{'DeferredFilesystemLock': <type at remote 0x92691b4>, 'AlreadyCalledError': <type at remote 0x912420c>, 'returnValue': <function at remote 0x9227684>, '_deferGenerator': <function at remote 0x92275a4>, 'DebugInfo': <classobj at remote 0x922032c>, '_parseDListResult': <function at remote 0x922702c>, 'FirstError': <type at remote 0x91253a4>, 'deferredGenerator': <function at remote 0x922764c>, 'fail': <function at remote 0x9217e64>, 'waitForDeferred': <classobj at remote 0x922044c>, 'inlineCallbacks': <function at remote 0x922772c>, 'logError': <function at remote 0x9217ca4>, 'log': <module at remote 0xb6ff5a64>, '__all__': ['Deferred', 'DeferredList', 'succeed', 'fail', 'FAILURE', 'SUCCESS', 'AlreadyCalledError', 'TimeoutError', 'gatherResults', 'maybeDeferred', 'waitForDeferred', 'deferredGenerator', 'inlineCallbacks', 'returnValue', 'DeferredLock', 'DeferredSemaphore', 'DeferredQueue', 'DeferredFilesystemLock', 'AlreadyTryingToLockError'], 'CancelledError': <type at remote 0x91243e4>, '__package__': 'twiste...(truncated), locals=0x0, args=0xa2fd8d8, argcount=2, kws=0xb73a1038, kwcount=0, defs=0x0,
defcount=0, closure=0x0) at Python/ceval.c:3253
(past this point in the backtrace, you get alot of python tracebacks ( something in the core file is messing with the python-gdb extension )
na = 0
nk = 0
n = 0
pfunc = 0xa0bf47c
func = <classobj at remote 0x931c9bc>
x =
<Failure(count=60, tb=None, value=exceptions.SystemError('Python/traceback.c:89: bad argument to internal function',), parents=['exceptions.SystemError', 'exceptions.StandardError', 'exceptions.Exception', 'exceptions.BaseException', '__builtin__.object', 'exceptions.SystemError'], frames=[], type=<type at remote 0xb76ee1e0>, stack=[]) at remote 0xa2fdb8c>
w = 0x0
I don't know if this will be helpful?
http://mail.python.org/pipermail/python-dev/2001-October/018002.html |
➜
