Projects / PersonDetection / perdet / utils / fastlz.py

# encoding: utf-8
# Author: Nicholas Piël
#
# A fast compressor and decompressor that will load and an array to file


import ctypes
import numpy
from perdet.config import *
    

FLZ = ctypes.cdll.LoadLibrary(binfile('fastlz.so'))
#FLZ = ctypes.cdll.LoadLibrary('/Users/nlp/Documents/school/Afstuderen/PersonDetection/Resources/bin/fastlz.so')


# We will be using the following header: 
#   dtype  size  shape_sizes

def load_compressed_array(filename):
    ''' Simply load the compressed array from filename
        use ctypes pointers so we do not need to copy the data
    '''
    f = open(filename, 'rb')
    header = f.readline().strip().split()
    dtype = header[0]
    size = int(header[1])
    shape = tuple([int(i) for i in header[2:]])
    data = decompress(f.read(), size-1)
    f.close()
    return numpy.fromstring(data, dtype=dtype).reshape(shape)


def save_compressed_array(filename, array):
    ''' Compress the array and save it to file
        XXX: could use ctypes pointers
    '''
    f = open(filename, 'wb')
    shape = ' '.join([str(i) for i in array.shape])
    header = "%s %s %s\n" % (array.dtype, array.nbytes, shape)
    f.write(header)
    f.write(compress(array.tostring()))
    f.close()
    


def compress(data):
    ''' This will compress the datastring and return the compressed 
        string
    '''
    datalen = len(data)
    output = ctypes.create_string_buffer('\000' * int(datalen * 1.05))
    count = FLZ.fastlz_compress(data, datalen, output)
    return output[:count]


def decompress(data, size=307200):
    ''' This will decompress the datastring and return the uncompressed
        data
    '''
    output = ctypes.create_string_buffer('\000' * size)
    FLZ.fastlz_decompress(data, size+1, output, len(output) )
    return output.raw


if __name__ == "__main__":
    import numpy
    a = numpy.ones((10,10)).astype('uint8')
    print "Saving..."
    save_compressed_array('array.flz', a)
    print "loading"
    b = load_compressed_array('array.flz')
    print b.shape, b.dtype, b
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.