Author nyevik
Recipients nyevik
Date 2011-03-15.22:58:27
SpamBayes Score 1.07303e-13
Marked as misclassified No
Message-id <1300229909.71.0.853987934234.issue11564@psf.upfronthosting.co.za>
In-reply-to
Content
Python 3.2 on linux (RHEL 5.3) x86_64 build from source code.
Configure options:
./configure --prefix=/scratch/Python-3.2 --enable-big-digits=30 --with-universal-archs=all --with-fpectl --enable-shared
Built with GCC 4.3.3 with major options 
-g3 -O3 -m64 -fPIC.

Testcase that shows the issue:

#import numpy

import pickle
print("begin")
#a = numpy.zeros((2.5e9 / 8,), dtype = numpy.float64)
a = ('a' * (2 ** 31))
print("allocated")
#print(a);
pickle.dumps(a, pickle.DEFAULT_PROTOCOL)
print("end")

The problem as I can see it is that in pickle.py most types defined either as 2 bytes, or 4 bytes.
For example it is peppered with lines like:
self.write(SOMETYPE + pack("<i", n) + obj)
while pickling,
when unpickling:
len = mloads('i' + self.read(4))

Which limits the range and the size of the datatype that can be pickled, if I understand correctly.

replacing in pickle.py above  lines with something like 
  self.write(SOMETYPE + pack("<Q", n) + obj)
  and
len = mloads('Q' + self.read(8))

lets above testcase run to completion.
Othervise it crashes (on Python 2.7.1 with SIGSEGV) on Python 3.2 strace shows:

.......


 open("/scratch/Python-3.2/lib/python3.2/lib-dynload/_pickle.cpython-32m.so", O_RDONLY) = 4
fstat(4, {st_mode=S_IFREG|0755, st_size=412939, ...}) = 0
open("/scratch/hpl005/UIT_test/apps_exc/Python-3.2/lib/python3.2/lib-dynload/_pickle.cpython-32m.so", O_RDONLY) = 5
read(5, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300>\0\0\0\0\0\0"..., 832) = 832
fstat(5, {st_mode=S_IFREG|0755, st_size=412939, ...}) = 0
mmap(NULL, 2185384, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 5, 0) = 0x2b05b5f68000
mprotect(0x2b05b5f7b000, 2093056, PROT_NONE) = 0
mmap(0x2b05b617a000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 5, 0x12000) = 0x2b05b617a000
close(5)                                = 0
close(4)                                = 0
close(3)                                = 0
write(1, "begin\n", 6begin
)                  = 6
mmap(NULL, 4294971392, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b05b617e000
write(1, "allocated\n", 10allocated
)             = 10
mmap(NULL, 8589938688, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b06b617f000
mremap(0x2b06b617f000, 8589938688, 2147487744, MREMAP_MAYMOVE) = 0x2b06b617f000
mmap(NULL, 4294971392, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b0736180000
munmap(0x2b06b617f000, 2147487744)      = 0
munmap(0x2b0736180000, 4294971392)      = 0
write(2, "Traceback (most recent call last"..., 35Traceback (most recent call last):
) = 35
write(2, "  File \"pickle_long.py\", line 9,"..., 45  File "pickle_long.py", line 9, in <module>
) = 45
open("pickle_long.py", O_RDONLY)        = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=251, ...}) = 0
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7ffff9f7c9e0) = -1 ENOTTY (Inappropriate ioctl for device)
fstat(3, {st_mode=S_IFREG|0644, st_size=251, ...}) = 0
lseek(3, 0, SEEK_CUR)                   = 0
dup(3)                                  = 4
fcntl(4, F_GETFL)                       = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fstat(4, {st_mode=S_IFREG|0644, st_size=251, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b06b617f000
lseek(4, 0, SEEK_CUR)                   = 0
read(4, "#import numpy\n\nimport pickle\npri"..., 4096) = 251
close(4)                                = 0
munmap(0x2b06b617f000, 4096)            = 0
lseek(3, 0, SEEK_SET)                   = 0
lseek(3, 0, SEEK_CUR)                   = 0
read(3, "#import numpy\n\nimport pickle\npri"..., 4096) = 251
close(3)                                = 0
write(2, "    pickle.dumps(a, pickle.DEFAU"..., 45    pickle.dumps(a, pickle.DEFAULT_PROTOCOL)
) = 45
write(2, "SystemError: error return withou"..., 48SystemError: error return without exception set
) = 48
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x2b05b118e4c0}, {0x2b05b0e7a570, [], SA_RESTORER, 0x2b05b118e4c0}, 8) = 0
munmap(0x2b05b617e000, 4294971392)      = 0
exit_group(1)                           = ?

Why is this limitation?
Please advise.
History
Date User Action Args
2011-03-15 22:58:30nyeviksetrecipients: + nyevik
2011-03-15 22:58:29nyeviksetmessageid: <1300229909.71.0.853987934234.issue11564@psf.upfronthosting.co.za>
2011-03-15 22:58:27nyeviklinkissue11564 messages
2011-03-15 22:58:27nyevikcreate