Commits

Romain Guillebert committed 33e7a4a Merge

Merge numpy-pickle

Comments (0)

Files changed (13)

pypy/doc/whatsnew-head.rst

 .. this is a revision shortly after release-2.0
 .. startrev: a13c07067613
 
+.. branch: numpy-pickle
+Pickling of numpy arrays and dtypes (including record dtypes)

pypy/interpreter/mixedmodule.py

     # imported yet, and when it has been, it is mod.__dict__.items() just
     # after startup().
     w_initialdict = None
+    lazy = False
 
     def __init__(self, space, w_name):
         """ NOT_RPYTHON """

pypy/module/micronumpy/__init__.py

         'zeros': 'interp_numarray.zeros',
         'empty': 'interp_numarray.zeros',
         'ones': 'interp_numarray.ones',
+        '_reconstruct' : 'interp_numarray._reconstruct',
         'dot': 'interp_arrayops.dot',
         'fromstring': 'interp_support.fromstring',
         'flatiter': 'interp_flatiter.W_FlatIterator',

pypy/module/micronumpy/arrayimpl/concrete.py

     def get_size(self):
         return self.size // self.dtype.itemtype.get_element_size()
 
+    def get_storage_size(self):
+        return self.size
+
     def reshape(self, space, orig_array, new_shape):
         # Since we got to here, prod(new_shape) == self.size
         new_strides = None
 
 
 class ConcreteArray(ConcreteArrayNotOwning):
-    def __init__(self, shape, dtype, order, strides, backstrides):
-        # we allocate the actual storage later because we need to compute
-        # self.size first
+    def __init__(self, shape, dtype, order, strides, backstrides, storage=lltype.nullptr(RAW_STORAGE)):
         null_storage = lltype.nullptr(RAW_STORAGE)
         ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides,
                                         null_storage)
-        self.storage = dtype.itemtype.malloc(self.size)
+        if storage == lltype.nullptr(RAW_STORAGE):
+            self.storage = dtype.itemtype.malloc(self.size)
+        else:
+            self.storage = storage
 
     def __del__(self):
         free_raw_storage(self.storage, track_allocation=False)

pypy/module/micronumpy/base.py

         return W_NDimArray(impl)
 
     @staticmethod
-    def from_shape_and_storage(shape, storage, dtype, order='C'):
+    def from_shape_and_storage(shape, storage, dtype, order='C', owning=False):
         from pypy.module.micronumpy.arrayimpl import concrete
         assert shape
         strides, backstrides = calc_strides(shape, dtype, order)
-        impl = concrete.ConcreteArrayNotOwning(shape, dtype, order, strides,
-                                               backstrides, storage)
+        if owning:
+            # Will free storage when GCd
+            impl = concrete.ConcreteArray(shape, dtype, order, strides,
+                                                backstrides, storage=storage)
+        else:
+            impl = concrete.ConcreteArrayNotOwning(shape, dtype, order, strides,
+                                                backstrides, storage)
         return W_NDimArray(impl)
 
     @staticmethod

pypy/module/micronumpy/interp_dtype.py

                                                                  space.wrap(offset)]))
         return w_d
 
+    def set_fields(self, space, w_fields):
+        if w_fields == space.w_None:
+            self.fields = None
+        else:
+            ofs_and_items = []
+            size = 0
+            for key in space.listview(w_fields):
+                value = space.getitem(w_fields, key)
+
+                dtype = space.getitem(value, space.wrap(0))
+                assert isinstance(dtype, W_Dtype)
+
+                offset = space.int_w(space.getitem(value, space.wrap(1)))
+                self.fields[space.str_w(key)] = offset, dtype
+
+                ofs_and_items.append((offset, dtype.itemtype))
+                size += dtype.itemtype.get_element_size()
+
+            self.itemtype = types.RecordType(ofs_and_items, size)
+            self.name = "void" + str(8 * self.itemtype.get_element_size())
+
     def descr_get_names(self, space):
         if self.fieldnames is None:
             return space.w_None
         return space.newtuple([space.wrap(name) for name in self.fieldnames])
 
+    def set_names(self, space, w_names):
+        if w_names == space.w_None:
+            self.fieldnames = None
+        else:
+            self.fieldnames = []
+            iter = space.iter(w_names)
+            while True:
+                try:
+                    self.fieldnames.append(space.str_w(space.next(iter)))
+                except OperationError, e:
+                    if not e.match(space, space.w_StopIteration):
+                        raise
+                    break
+
     @unwrap_spec(item=str)
     def descr_getitem(self, space, item):
         if self.fields is None:
     def get_size(self):
         return self.itemtype.get_element_size()
 
+    def descr_reduce(self, space):
+        w_class = space.type(self)
+
+        kind = self.kind
+        elemsize = self.itemtype.get_element_size()
+        builder_args = space.newtuple([space.wrap("%s%d" % (kind, elemsize)), space.wrap(0), space.wrap(1)])
+
+        version = space.wrap(3)
+        order = space.wrap(byteorder_prefix if self.native else nonnative_byteorder_prefix)
+        names = self.descr_get_names(space)
+        values = self.descr_get_fields(space)
+        if self.fields:
+            #TODO: Implement this when subarrays are implemented
+            subdescr = space.w_None
+            #TODO: Change this when alignment is implemented :
+            size = 0
+            for key in self.fields:
+                dtype = self.fields[key][1]
+                assert isinstance(dtype, W_Dtype)
+                size += dtype.get_size()
+            w_size = space.wrap(size)
+            alignment = space.wrap(1)
+        else:
+            subdescr = space.w_None
+            w_size = space.wrap(-1)
+            alignment = space.wrap(-1)
+        flags = space.wrap(0)
+
+        data = space.newtuple([version, order, subdescr, names, values, w_size, alignment, flags])
+
+        return space.newtuple([w_class, builder_args, data])
+
+    def descr_setstate(self, space, w_data):
+        if space.int_w(space.getitem(w_data, space.wrap(0))) != 3:
+            raise OperationError(space.w_NotImplementedError, space.wrap("Pickling protocol version not supported"))
+
+        self.native = space.str_w(space.getitem(w_data, space.wrap(1))) == byteorder_prefix
+
+        fieldnames = space.getitem(w_data, space.wrap(3))
+        self.set_names(space, fieldnames)
+
+        fields = space.getitem(w_data, space.wrap(4))
+        self.set_fields(space, fields)
+        print self.itemtype
+
 class W_ComplexDtype(W_Dtype):
     def __init__(self, itemtype, num, kind, name, char, w_box_type,
                  alternate_constructors=[], aliases=[],
         num = 20
         basename = 'void'
         w_box_type = space.gettypefor(interp_boxes.W_VoidBox)
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "pure void dtype"))
+        return dtype_from_list(space, space.newlist([]))
     else:
         assert char == 'U'
         basename = 'unicode'
 
 def dtype_from_spec(space, name):
         raise OperationError(space.w_NotImplementedError, space.wrap(
-            "dtype from spec"))    
+            "dtype from spec"))
 
-def descr__new__(space, w_subtype, w_dtype):
+def descr__new__(space, w_subtype, w_dtype, w_align=None, w_copy=None):
+    # w_align and w_copy are necessary for pickling
     cache = get_dtype_cache(space)
 
     if space.is_none(w_dtype):
     __ne__ = interp2app(W_Dtype.descr_ne),
     __getitem__ = interp2app(W_Dtype.descr_getitem),
 
+    __reduce__ = interp2app(W_Dtype.descr_reduce),
+    __setstate__ = interp2app(W_Dtype.descr_setstate),
+
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
     char = interp_attrproperty("char", cls=W_Dtype),

pypy/module/micronumpy/interp_numarray.py

             return space.float(self.descr_getitem(space, space.wrap(0)))
         raise OperationError(space.w_TypeError, space.wrap("only length-1 arrays can be converted to Python scalars"))
 
+    def descr_reduce(self, space):
+        from rpython.rtyper.lltypesystem import rffi
+        from rpython.rlib.rstring import StringBuilder
+        from pypy.interpreter.mixedmodule import MixedModule
+
+        numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(numpypy, MixedModule)
+        multiarray = numpypy.get("multiarray")
+        assert isinstance(multiarray, MixedModule)
+        reconstruct = multiarray.get("_reconstruct")
+
+        parameters = space.newtuple([space.gettypefor(W_NDimArray), space.newtuple([space.wrap(0)]), space.wrap("b")])
+
+        builder = StringBuilder()
+        builder.append_charpsize(self.implementation.get_storage(), self.implementation.get_storage_size())
+
+        state = space.newtuple([
+                space.wrap(1),      # version
+                self.descr_get_shape(space),
+                self.get_dtype(),
+                space.wrap(False),  # is_fortran
+                space.wrap(builder.build()),
+            ])
+
+        return space.newtuple([reconstruct, parameters, state])
+
+    def descr_setstate(self, space, w_state):
+        from rpython.rtyper.lltypesystem import rffi
+
+        shape = space.getitem(w_state, space.wrap(1))
+        dtype = space.getitem(w_state, space.wrap(2))
+        assert isinstance(dtype, interp_dtype.W_Dtype)
+        isfortran = space.getitem(w_state, space.wrap(3))
+        storage = space.getitem(w_state, space.wrap(4))
+
+        self.implementation = W_NDimArray.from_shape_and_storage([space.int_w(i) for i in space.listview(shape)], rffi.str2charp(space.str_w(storage), track_allocation=False), dtype, owning=True).implementation
 
 
 @unwrap_spec(offset=int)
 
 W_NDimArray.typedef = TypeDef(
     "ndarray",
+    __module__ = "numpypy",
     __new__ = interp2app(descr_new_array),
 
     __len__ = interp2app(W_NDimArray.descr_len),
     __pypy_data__ = GetSetProperty(W_NDimArray.fget___pypy_data__,
                                    W_NDimArray.fset___pypy_data__,
                                    W_NDimArray.fdel___pypy_data__),
+    __reduce__ = interp2app(W_NDimArray.descr_reduce),
+    __setstate__ = interp2app(W_NDimArray.descr_setstate),
 )
 
 @unwrap_spec(ndmin=int, copy=bool, subok=bool)
     arr.fill(one)
     return space.wrap(arr)
 
+def _reconstruct(space, w_subtype, w_shape, w_dtype):
+    return descr_new_array(space, w_subtype, w_shape, w_dtype)
+
 W_FlatIterator.typedef = TypeDef(
     'flatiter',
     __iter__ = interp2app(W_FlatIterator.descr_iter),

pypy/module/micronumpy/test/test_base.py

 
         # Coerce to floats, some of these will eventually be float16, or
         # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float16_dtype 
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float16_dtype 
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float16_dtype
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float16_dtype
         assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float16_dtype
         assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype

pypy/module/micronumpy/test/test_dtypes.py

             from rpython.rtyper.lltypesystem import rffi
             ptr_size = rffi.sizeof(rffi.CCHARP)
         cls.w_ptr_size = cls.space.wrap(ptr_size)
-    
+
 class AppTestDtypes(BaseAppTestDtypes):
+    spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
     def test_dtype(self):
         from numpypy import dtype
 
     def test_bool_binop_types(self):
         from numpypy import array, dtype
         types = [
-            '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd', 
+            '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd',
             'e'
         ]
         a = array([True], '?')
         ]:
             assert hash(tp(value)) == hash(value)
 
+    def test_pickle(self):
+        from numpypy import array, dtype
+        from cPickle import loads, dumps
+        a = array([1,2,3])
+        assert a.dtype.__reduce__() == (dtype, ('i8', 0, 1), (3, '<', None, None, None, -1, -1, 0))
+        assert loads(dumps(a.dtype)) == a.dtype
+
+    def test_pickle_record(self):
+        from numpypy import array, dtype
+        from cPickle import loads, dumps
+
+        d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
+        assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '<', None, ('x', 'y', 'z', 'value'), {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0), 'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12)}, 20, 1, 0))
+
+        new_d = loads(dumps(d))
+
+        assert new_d.__reduce__() == d.__reduce__()
 
 class AppTestTypes(BaseAppTestDtypes):
     def test_abstract_types(self):
         import numpypy as numpy
 
         assert numpy.int8.mro() == [numpy.int8, numpy.signedinteger,
-                                    numpy.integer, numpy.number, 
+                                    numpy.integer, numpy.number,
                                     numpy.generic, object]
 
         a = numpy.array([1, 2, 3], numpy.int8)
     def test_uint8(self):
         import numpypy as numpy
 
-        assert numpy.uint8.mro() == [numpy.uint8, numpy.unsignedinteger, 
-                                     numpy.integer, numpy.number, 
+        assert numpy.uint8.mro() == [numpy.uint8, numpy.unsignedinteger,
+                                     numpy.integer, numpy.number,
                                      numpy.generic, object]
 
         a = numpy.array([1, 2, 3], numpy.uint8)
         import numpypy as numpy
 
         assert numpy.int_ is numpy.dtype(int).type
-        assert numpy.int_.mro() == [numpy.int_, numpy.signedinteger, 
-                                    numpy.integer, numpy.number, 
+        assert numpy.int_.mro() == [numpy.int_, numpy.signedinteger,
+                                    numpy.integer, numpy.number,
                                     numpy.generic, int, object]
 
     def test_int64(self):
         import numpypy as numpy
 
         if sys.maxint == 2 ** 63 -1:
-            assert numpy.int64.mro() == [numpy.int64, numpy.signedinteger, 
-                                         numpy.integer, numpy.number, 
+            assert numpy.int64.mro() == [numpy.int64, numpy.signedinteger,
+                                         numpy.integer, numpy.number,
                                          numpy.generic, int, object]
         else:
-            assert numpy.int64.mro() == [numpy.int64, numpy.signedinteger, 
-                                         numpy.integer, numpy.number, 
+            assert numpy.int64.mro() == [numpy.int64, numpy.signedinteger,
+                                         numpy.integer, numpy.number,
                                          numpy.generic, object]
 
         assert numpy.dtype(numpy.int64).type is numpy.int64
         import sys
         import numpypy as numpy
 
-        assert numpy.uint64.mro() == [numpy.uint64, numpy.unsignedinteger, 
-                                      numpy.integer, numpy.number, 
+        assert numpy.uint64.mro() == [numpy.uint64, numpy.unsignedinteger,
+                                      numpy.integer, numpy.number,
                                       numpy.generic, object]
 
         assert numpy.dtype(numpy.uint64).type is numpy.uint64
 
     def test_float16(self):
         import numpypy as numpy
-        assert numpy.float16.mro() == [numpy.float16, numpy.floating, 
-                                       numpy.inexact, numpy.number, 
+        assert numpy.float16.mro() == [numpy.float16, numpy.floating,
+                                       numpy.inexact, numpy.number,
                                        numpy.generic, object]
 
         assert numpy.float16(12) == numpy.float64(12)
     def test_float32(self):
         import numpypy as numpy
 
-        assert numpy.float32.mro() == [numpy.float32, numpy.floating, 
-                                       numpy.inexact, numpy.number, 
+        assert numpy.float32.mro() == [numpy.float32, numpy.floating,
+                                       numpy.inexact, numpy.number,
                                        numpy.generic, object]
 
         assert numpy.float32(12) == numpy.float64(12)
     def test_float64(self):
         import numpypy as numpy
 
-        assert numpy.float64.mro() == [numpy.float64, numpy.floating, 
-                                       numpy.inexact, numpy.number, 
+        assert numpy.float64.mro() == [numpy.float64, numpy.floating,
+                                       numpy.inexact, numpy.number,
                                        numpy.generic, float, object]
 
         a = numpy.array([1, 2, 3], numpy.float64)
         # it can be float96 or float128
         if numpy.longfloat != numpy.float64:
             assert numpy.longfloat.mro()[1:] == [numpy.floating,
-                                       numpy.inexact, numpy.number, 
+                                       numpy.inexact, numpy.number,
                                        numpy.generic, object]
         a = numpy.array([1, 2, 3], numpy.longdouble)
         assert type(a[1]) is numpy.longdouble
         a = array([1, 2, 3], dtype=self.non_native_prefix + 'G') # clongdouble
         assert a[0] == 1
         assert (a + a)[1] == 4
+

pypy/module/micronumpy/test/test_numarray.py

         assert get(1, 1) == 3
 
 class AppTestNumArray(BaseNumpyAppTest):
+    spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
     def w_CustomIndexObject(self, index):
         class CustomIndexObject(object):
             def __init__(self, index):
         assert raises(TypeError, "int(array([1, 2]))")
         assert int(array([1.5])) == 1
 
+    def test__reduce__(self):
+        from numpypy import array, dtype
+        from cPickle import loads, dumps
+
+        a = array([1, 2], dtype="int64")
+        data = a.__reduce__()
+
+        assert data[2][4] == '\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00'
+
+        pickled_data = dumps(a)
+        assert (loads(pickled_data) == a).all()
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
 
 
 class AppTestRecordDtype(BaseNumpyAppTest):
+    spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
+
     def test_zeros(self):
         from numpypy import zeros, integer
         a = zeros(2, dtype=[('x', int), ('y', float)])
         assert s.replace('\n', '') == \
                       "array(['abc', 'defg', 'ab'],       dtype='|S4')"
 
+    def test_pickle(self):
+        from numpypy import dtype, array
+        from cPickle import loads, dumps
+
+        d = dtype([('x', str), ('y', 'int32')])
+        a = array([('a', 2), ('cde', 1)], dtype=d)
+
+        a = loads(dumps(a))
+        d = a.dtype
+
+        assert str(d.fields['x'][0]) == '|S0'
+        assert d.fields['x'][1] == 0
+        assert str(d.fields['y'][0]) == 'int32'
+        assert d.fields['y'][1] == 0
+        assert d.name == 'void32'
+
+        assert a[0]['y'] == 2
+        assert a[1]['y'] == 1
+
 
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
Add a comment to this file

pypy/module/test_lib_pypy/ctypes_tests/conftest.py

File contents unchanged.

Add a comment to this file

rpython/jit/backend/x86/test/test_exception.py

File contents unchanged.

rpython/rtyper/lltypesystem/rffi.py

     TYPES += ['__int128_t']
 except CompilationError:
     pass
-    
+
 _TYPES_ARE_UNSIGNED = set(['size_t', 'uintptr_t'])   # plus "unsigned *"
 if os.name != 'nt':
     TYPES.append('mode_t')
         builder_class = UnicodeBuilder
 
     # str -> char*
-    def str2charp(s):
+    def str2charp(s, track_allocation=True):
         """ str -> char*
         """
-        array = lltype.malloc(TYPEP.TO, len(s) + 1, flavor='raw')
+        if track_allocation:
+            array = lltype.malloc(TYPEP.TO, len(s) + 1, flavor='raw', track_allocation=True)
+        else:
+            array = lltype.malloc(TYPEP.TO, len(s) + 1, flavor='raw', track_allocation=False)
         i = len(s)
         array[i] = lastchar
         i -= 1
             array[i] = s[i]
             i -= 1
         return array
-    str2charp._annenforceargs_ = [strtype]
+    str2charp._annenforceargs_ = [strtype, bool]
 
-    def free_charp(cp):
-        lltype.free(cp, flavor='raw')
+    def free_charp(cp, track_allocation=True):
+        if track_allocation:
+            lltype.free(cp, flavor='raw', track_allocation=True)
+        else:
+            lltype.free(cp, flavor='raw', track_allocation=False)
 
     # char* -> str
     # doesn't free char*
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.