Message201295
in python 2.7.3 this kind of works however it is less efficient than the pure lzma.compress()
from threading import Thread
from backports import lzma
from functools import partial
import multiprocessing
class CompressClass(Thread):
def __init__ (self,data,c):
Thread.__init__(self)
self.exception=False
self.data=data
self.datacompressed=""
self.c=c
def getException(self):
return self.exception
def getOutput(self):
return self.datacompressed
def run(self):
self.datacompressed=(self.c).compress(self.data)
def split_len(seq, length):
return [seq[i:i+length] for i in range(0, len(seq), length)]
def launch_multiple_lzma(data,c):
print 'cores'
present=CompressClass(data,c)
present.start()
present.join()
return present.getOutput()
def threaded_lzma_map(sequence,threads):
lzc = lzma.LZMACompressor()
blocksize = int(round(len(sequence)/threads))
lzc_partial = partial(launch_multiple_lzma,c=lzc)
lzlist = map(lzc_partial,split_len(sequence, blocksize))
#pool=multiprocessing.Pool()
#lzclist = pool.map(lzc_partial,split_len(sequence, blocksize))
#pool.close()
#pool.join()
out_flush = lzc.flush()
res = "".join(lzlist + [out_flush])
return res
sequence = 'AAAAAJKDDDDDDDDDDDDDDDDDDDDDDDDDDDDGJFKSHFKLHALWEHAIHWEOIAH IOAHIOWEHIOHEIOFEAFEASFEAFWEWWWWWWWWWWWWWWWWWWWWWWWWWWWWWEWFQWEWQWQGEWQFEWFDWEWEGEFGWEG'
lzma.compress(sequence) == threaded_lzma_map(sequence,threads=16)
Any way this could be imporved? |
|
Date |
User |
Action |
Args |
2013-10-25 21:23:18 | cantor | set | recipients:
+ cantor, pitrou, nadeem.vawda |
2013-10-25 21:23:18 | cantor | set | messageid: <1382736198.66.0.19571054853.issue19395@psf.upfronthosting.co.za> |
2013-10-25 21:23:18 | cantor | link | issue19395 messages |
2013-10-25 21:23:18 | cantor | create | |
|