This issue tracker has been migrated to GitHub, and is currently read-only.
For more information, see the GitHub FAQs in the Python's Developer Guide.

Author cantor
Recipients cantor, nadeem.vawda, pitrou
Date 2013-10-25.21:23:18
SpamBayes Score -1.0
Marked as misclassified Yes
Message-id <1382736198.66.0.19571054853.issue19395@psf.upfronthosting.co.za>
In-reply-to
Content
in python 2.7.3  this kind of works however it is less efficient than the pure lzma.compress()

from threading import Thread
from backports import lzma
from functools import partial
import multiprocessing


class CompressClass(Thread):
  def __init__ (self,data,c):
    Thread.__init__(self)
    self.exception=False
    self.data=data
    self.datacompressed=""
    self.c=c
  def getException(self):
    return self.exception   
  def getOutput(self):
    return self.datacompressed
  def run(self):
        self.datacompressed=(self.c).compress(self.data)


def split_len(seq, length):
    return [seq[i:i+length] for i in range(0, len(seq), length)]



def launch_multiple_lzma(data,c):
    print 'cores'
    present=CompressClass(data,c) 
    present.start()      
    present.join()
    return present.getOutput()


def threaded_lzma_map(sequence,threads):
  lzc = lzma.LZMACompressor()
  blocksize = int(round(len(sequence)/threads))
  lzc_partial = partial(launch_multiple_lzma,c=lzc)
  lzlist = map(lzc_partial,split_len(sequence, blocksize))
  #pool=multiprocessing.Pool()
  #lzclist = pool.map(lzc_partial,split_len(sequence, blocksize))
  #pool.close()
  #pool.join()
  out_flush = lzc.flush()
  res = "".join(lzlist + [out_flush])
  return res 

sequence = 'AAAAAJKDDDDDDDDDDDDDDDDDDDDDDDDDDDDGJFKSHFKLHALWEHAIHWEOIAH IOAHIOWEHIOHEIOFEAFEASFEAFWEWWWWWWWWWWWWWWWWWWWWWWWWWWWWWEWFQWEWQWQGEWQFEWFDWEWEGEFGWEG'

lzma.compress(sequence) == threaded_lzma_map(sequence,threads=16)

Any way this could be imporved?
History
Date User Action Args
2013-10-25 21:23:18cantorsetrecipients: + cantor, pitrou, nadeem.vawda
2013-10-25 21:23:18cantorsetmessageid: <1382736198.66.0.19571054853.issue19395@psf.upfronthosting.co.za>
2013-10-25 21:23:18cantorlinkissue19395 messages
2013-10-25 21:23:18cantorcreate