Commits

Daniel Roberts committed fc5c110

Cache offset into array data, passing way more tests.

Comments (0)

Files changed (5)

pypy/module/micronumpy/array.py

     return stride
 
 def stride_column(shape, i):
+    assert i >= 0
     i -= 1
     stride = 1
     while i >= 0:
 
 def size_from_shape(shape):
     size = 1
-    for dimension in shape:
-        size *= dimension
-    return size
+    if len(shape) > 0:
+        for dimension in shape:
+            size *= dimension
+        return size
+    else:
+        return 0
 
 def normalize_slice_starts(slice_starts, shape):
     for i in range(len(slice_starts)):
-        #print "slice_start[%d]=%d" % (i, slice_starts[i])
         if slice_starts[i] < 0:
             slice_starts[i] += shape[i]
         elif slice_starts[i] >= shape[i]:
-            print "raising"
             raise IndexError("invalid index")
     return slice_starts
 
                 space.wrap("invalid index")) # all as in numpy
 
 def infer_shape(space, w_values):
+    try:
+        values = space.str_w(w_values)
+        return [len(values)]
+    except OperationError, e:
+        if e.match(space, space.w_TypeError): pass
+        else: raise
+
     shape = []
     while True:
         try:

pypy/module/micronumpy/dtype.py

         def cast(self, data):
             return rffi.cast(lltype.Ptr(arraytype), data)
 
-        def dump(self, data):
-            return ', '.join([str(x) for x in self.cast(data)])
+        def dump(self, data, count):
+            data = self.cast(data)
+            return ', '.join([str(data[i]) for i in range(count)])
 
         def typestr(self):
             if self is float_descr:
 # i is ??
 
 int_descr = descriptor('i', 'int32', lltype.Signed)
-type(int_descr).unwrap = lambda self, space, value: space.int_w(value)
-type(int_descr).coerce_w = lambda self, space, value: space.int_w(space.int(value))
+IntDescrImpl = type(int_descr)
+IntDescrImpl.unwrap = lambda self, space, value: space.int_w(value)
+IntDescrImpl.coerce = lambda self, space, value: space.int(value)
+IntDescrImpl.coerce_w = lambda self, space, value: space.int_w(space.int(value))
 _int_index = _typeindex['i']
 _typestring['int32'] = _int_index
 w_int_descr = _w_descriptors[_int_index]
 
 float_descr = descriptor('d', 'float64', lltype.Float)
-type(float_descr).unwrap = lambda self, space, value: space.float_w(value)
-type(float_descr).coerce_w = lambda self, space, value: space.float_w(space.float(value))
+FloatDescrImpl = type(float_descr)
+FloatDescrImpl.unwrap = lambda self, space, value: space.float_w(value)
+FloatDescrImpl.coerce = lambda self, space, value: space.float(value)
+FloatDescrImpl.coerce_w = lambda self, space, value: space.float_w(space.float(value))
 _float_index = _typeindex['d']
 _typestring['float64'] = _float_index
 w_float_descr = _w_descriptors[_float_index]

pypy/module/micronumpy/microarray.py

 
 from pypy.rpython.lltypesystem.lltype import cast_ptr_to_int
 
-class FlatIter(Wrappable):
-    _immutable_fields_ = ['array', 'stop']
+class MicroIter(Wrappable):
+    _immutable_fields_ = ['array', 'offset', 'step', 'shape', 'ndim']
     def __init__(self, array):
         self.array = array
         self.i = 0
-        self.stop = size_from_shape(array.shape)
-
-    def descr_iter(self, space):
-        return space.wrap(self)
-    descr_iter.unwrap_spec = ['self', ObjSpace]
-
-    def descr_next(self, space):
-        return self.array.getitem(space, self.i) # FIXME
-    descr_iter.unwrap_spec = ['self', ObjSpace]
-
-class MicroIter(Wrappable):
-    _immutable_fields_ = ['array', 'step', 'stop', 'ndim']
-    def __init__(self, array):
-        self.array = array
-        self.i = 0
-        self.index = array.slice_starts[:]
-        self.step = array.slice_steps[array.prefix]
-        self.stop = array.shape[array.prefix]
-        self.ndim = len(array.shape) - array.prefix
+        self.step = array.slice_steps[0]
+        self.shape = array.shape[0]
+        self.stride = array.strides[0]
+        self.ndim = len(array.shape)
+        self.offset = 0
 
     def descr_iter(self, space):
         return space.wrap(self)
     descr_iter.unwrap_spec = ['self', ObjSpace]
     
     def descr_next(self, space):
-        if self.i < self.stop:
-            print self.index
+        if self.i < self.shape:
             if self.ndim > 1:
                 ar = MicroArray(self.array.shape,
                                 self.array.dtype,
                                 parent=self.array,
-                                offset=self.array.prefix + 1,
-                                strides=self.array.strides,
-                                slice_starts=self.index,
-                                slice_steps=self.array.slice_steps)
+                                offset=self.offset + self.array.offset,
+                                strides=self.array.strides[1:],
+                                slice_steps=self.array.slice_steps[1:])
                 next = space.wrap(ar)
             elif self.ndim == 1:
-                next = self.array.getitem(space, self.array.flatten_slice_starts(self.index))
+                next = self.array.getitem(space, self.offset)
             else:
                 raise OperationError(space.w_ValueError,
                        space.wrap("Something is horribly wrong with this array's shape. Has %d dimensions." % len(self.array.shape)))
             self.i += 1
-            self.index[self.array.prefix] += self.step
+            self.offset += self.step * self.stride
             return next
         else:
             raise OperationError(space.w_StopIteration, space.wrap(""))
 
 
 class MicroArray(BaseNumArray):
-    _immutable_fields_ = ['shape', 'strides', 'offset', 'slice_starts'] # XXX: removed parent
+    _immutable_fields_ = ['shape', 'parent', 'strides', 'offset', 'slice_starts']
     def __init__(self, shape, dtype,
-                 order='C', strides=[], parent=None,
-                 prefix=0, offset=0,
-                 slice_starts=[], slice_steps=[]):
+                 order='C', strides=None, parent=None,
+                 offset=0, slice_steps=None):
         assert dtype is not None
 
         self.shape = shape
         self.parent = parent
         self.order = order
         self.offset = offset
-        self.prefix = prefix
 
-        self.slice_starts = slice_starts[:]
-        for i in range(len(slice_starts), len(shape)):
-            self.slice_starts.append(0)
+        if slice_steps is not None:
+            self.slice_steps = slice_steps
+        else:
+            self.slice_steps = []
 
-        self.slice_steps = slice_steps[:]
-        for i in range(len(slice_steps), len(shape)):
+        for i in range(len(self.slice_steps), len(shape)):
             self.slice_steps.append(1)
 
         size = size_from_shape(shape)
 
-        self.strides = strides[:]
+        if strides is not None:
+            self.strides = strides
+        else:
+            self.strides = []
+
         for i in range(len(self.strides), len(shape)):
             self.strides.append(self.stride(i))
 
             self.data = null_data
 
     def descr_len(self, space):
-        return space.wrap(self.shape[self.prefix])
+        return space.wrap(self.shape[0])
     descr_len.unwrap_spec = ['self', ObjSpace]
 
     def getitem(self, space, offset):
         """Helper function.
            Grabs a value at an offset into the data."""
         try:
-            return self.dtype.dtype.w_getitem(space, self.data, offset)
+            return self.dtype.dtype.w_getitem(space, self.data, self.offset + offset)
         except IndexError, e:
             raise OperationError(space.w_IndexError,
                                  space.wrap("index out of bounds"))
         """Helper function.
            Sets a value at an offset in the data."""
         try:
-            self.dtype.dtype.w_setitem(space, self.data, offset, w_value)
+            self.dtype.dtype.w_setitem(space, self.data, self.offset + offset, w_value)
         except IndexError, e:
             raise OperationError(space.w_IndexError,
                                  space.wrap("index out of bounds"))
 
-    def flatten_slice_starts(self, slice_starts):
-        """Computes offset into subarray from all information.
-           Gives offset into subarray, not into data."""
-        offset = 0
-        for i in range(len(slice_starts)):
-            offset += slice_starts[i] * self.strides[i]
-        #print offset
-        return offset
-
-    def flatten_slice_starts2(self, slice_starts):
-        """Computes offset into subarray from all information.
-           Gives offset into subarray, not into data."""
-        offset = 0
-        for i in range(len(slice_starts)):
-            offset += (self.slice_steps[i] * slice_starts[i]) * self.strides[i]
-        print offset
-        return offset
-
-    flatten_index = flatten_slice_starts2 # TODO: migrate to slice_starts for name?
-
     def stride(self, i):
         if self.order == 'C':
             return stride_row(self.shape, i) # row order for C
     def index2slices(self, space, w_index):
         dtype = self.dtype.dtype
 
-        slice_starts = self.slice_starts[:]
-        shape = self.shape[:]
-        slice_steps = self.slice_steps[:]
+        offset = 0
 
         try:
             index = space.int_w(space.index(w_index))
-            # Normalize 
             if index < 0:
-                index += self.shape[self.prefix]
-            elif index > self.shape[self.prefix]:
+                index += self.shape[0]
+            elif index >= self.shape[0]:
                 raise OperationError(space.w_IndexError,
-                                     space.wrap("invalid index")) # FIXME: message
+                                     space.wrap("index out of bounds"))
 
-            slice_starts[self.prefix] += index * self.slice_steps[self.prefix]
-            shape[self.prefix] = 1 #SQUEEZE_ME
-            return slice_starts, shape, slice_steps
+            offset = index * self.slice_steps[0] * self.strides[0]
+            return offset, self.shape[1:], self.slice_steps[1:], self.strides[1:]
         except OperationError, e:
             if e.match(space, space.w_TypeError): pass
             else:raise
 
         if isinstance(w_index, W_SliceObject):
             start, stop, step, length = w_index.indices4(space, self.shape[0])
-            slice_starts[self.prefix] += start * slice_steps[self.prefix]
-            shape[self.prefix] = length
-            slice_steps[self.prefix] *= step
-            return slice_starts, shape, slice_steps
+            offset = start * self.slice_steps[0] * self.strides[0]
+
+            shape = self.shape[:]
+            slice_steps = self.slice_steps[:]
+            shape[0] = length
+            slice_steps[0] *= step
+            return offset, shape, slice_steps, self.strides[:]
         elif space.is_w(w_index, space.w_Ellipsis):
-            return slice_starts, shape, slice_steps
+            return 0, self.shape[:], self.slice_steps[:], self.strides[:]
 
-        try:
-            indices = space.fixedview(w_index)
+        indices = space.fixedview(w_index)
+        
+        ndim = len(self.shape)
 
-            indexlen = len(indices)
-            if indexlen != len(self.shape): # FIXME: shape will often be larger...
-                raise OperationError(space.w_IndexError,
-                                     space.wrap("invalid index")) # FIXME: message
+        indexlen = len(indices)
+        if indexlen > ndim:
+            raise OperationError(space.w_IndexError,
+                                 space.wrap("invalid index"))
 
-            for i in range(indexlen):
-                w_index = indices[i]
-                try:
-                    index = space.int_w(space.index(w_index))
-                    slice_starts[self.prefix + i] += index
-                    shape[self.prefix + i] = 1 #SQUEEZE_ME
-                    continue
+        shape = [0] * ndim
+        strides = [0] * ndim
+        slice_steps = [0] * ndim
 
-                except OperationError, e:
-                    if e.match(space, space.w_TypeError): pass
-                    else: raise
+        resdim = 0
 
-                if isinstance(w_index, W_SliceObject):
-                    start, stop, step, length = w_index.indices4(space, self.shape[i])
-                    slice_starts[self.prefix + i] += start * slice_steps[self.prefix + i]
-                    shape[self.prefix + i] = length
-                    slice_steps[self.prefix + i] *= step
-                elif space.is_w(w_index, space.w_Ellipsis):
-                    pass # I can't think of anything we need to do
-                else:
-                    index = space.str(w_index)
-                    raise OperationError(space.w_ValueError,
-                                         space.wrap("Don't support records,"
-                                                    " so pretend we don't have this field"))
-                    raise OperationError(space.w_NotImplementedError, # this is the correct error
-                                         space.wrap("Don't support records yet.")) # for what we actually do
+        for i in range(indexlen):
+            w_index = indices[i]
+            try:
+                index = space.int_w(space.index(w_index))
+                if index < 0:
+                    index += self.shape[i]
+                elif index >= self.shape[i]:
+                    raise OperationError(space.w_IndexError,
+                                         space.wrap("index out of bounds"))
+                offset += index * self.slice_steps[i] * self.strides[i]
+                continue
 
-            try:
-                normalize_slice_starts(slice_starts, self.shape) # XXX: in-place operation
-            except IndexError, e:
-                raise OperationError(space.w_IndexError,
-                                     space.wrap("invalid index"))
-        finally: pass
+            except OperationError, e:
+                if e.match(space, space.w_TypeError): pass
+                else: raise
 
-        return slice_starts, shape, slice_steps
+            if isinstance(w_index, W_SliceObject):
+                start, stop, step, length = w_index.indices4(space, self.shape[i])
+                offset += start * self.slice_steps[i] * self.strides[i]
+                shape[resdim] = length
+                slice_steps[resdim] = self.slice_steps[i] * step
+                resdim += 1
+            elif space.is_w(w_index, space.w_Ellipsis):
+                shape[resdim] = self.shape[i]
+                slice_steps[resdim] = self.slice_steps[i]
+                strides[resdim] = self.strides[i]
+                resdim += 1
+            else:
+                index = space.str(w_index)
+                raise OperationError(space.w_ValueError,
+                                     space.wrap("Don't support records,"
+                                                " so pretend we don't have this field"))
+                raise OperationError(space.w_NotImplementedError, # this is the correct error
+                                     space.wrap("Don't support records yet.")) # for what we actually do
+
+        return offset, shape[:resdim], slice_steps[:resdim], strides[:resdim]
 
     def descr_getitem(self, space, w_index):
-        slice_starts, shape, slice_steps = self.index2slices(space, w_index)
+        offset, shape, slice_steps, strides = self.index2slices(space, w_index)
 
         size = size_from_shape(shape)
 
-        if size == 1:
-            return self.getitem(space,
-                                self.flatten_slice_starts(slice_starts))
+        if size == 0:
+            return self.getitem(space, offset)
         else:
-            prefix = shape_prefix(shape)
             ar = MicroArray(shape,
                             dtype=self.dtype,
                             parent=self,
-                            offset=prefix, # XXX: what do we do about shapes that needs to be squeezed out?
-                            strides=self.strides[:], 
-                            slice_starts=slice_starts,
+                            offset=self.offset + offset,
+                            strides=strides,
                             slice_steps=slice_steps)
             return space.wrap(ar)
     descr_getitem.unwrap_spec = ['self', ObjSpace, W_Root]
 
-    def set_slice_single_value(self, space, slice_starts, shape, slice_steps, w_value):
-        index = slice_starts[:]
+    def set_slice_single_value(self, space, offset, shape, slice_steps, strides, w_value):
         if len(shape) > 1:
             for i in range(shape[0]):
-                self.set_slice_single_value(space, index, shape[1:], slice_steps[1:], w_value)
-                index[len(index) - len(shape)] += slice_steps[0]
+                self.set_slice_single_value(space, offset, shape[1:], slice_steps[1:], strides[1:], w_value)
+                offset += slice_steps[0] * strides[0]
         else:
             for i in range(shape[0]):
-                self.setitem(space, self.flatten_index(index), w_value)
-                index[len(index)-1] += slice_steps[0]
+                self.setitem(space, offset, w_value)
+                offset += slice_steps[0] * strides[0]
 
-    def set_slice(self, space, slice_starts, shape, slice_steps, w_value):
+    def set_slice(self, space, offset, shape, slice_steps, strides, w_value):
         try:
             length = space.int_w(space.len(w_value))
 
             if length == 1:
-                self.set_slice_single_value(space, slice_starts, shape, slice_steps, w_value)
+                w_value = space.getitem(w_value, space.wrap(0))
+                self.set_slice_single_value(space, offset, shape, slice_steps, strides, w_value)
             else:
                 raise OperationError(space.w_NotImplementedError, # XXX: TODO
                                      space.wrap("TODO"))
         except OperationError, e:
             if e.match(space, space.w_TypeError):
-                self.set_slice_single_value(space, slice_starts, shape, slice_steps, w_value)
+                self.set_slice_single_value(space, offset, shape, slice_steps, strides, w_value)
             else: raise
 
     def descr_setitem(self, space, w_index, w_value):
         dtype = self.dtype.dtype
 
-        slice_starts, shape, slice_steps = self.index2slices(space, w_index)
+        offset, shape, slice_steps, strides = self.index2slices(space, w_index)
 
         size = size_from_shape(shape)
 
         try:
-            if space.int_w(space.len(w_value)) == 1:
-                w_value = space.getitem(w_value, space.wrap(0))
+            # XXX: if size is 0 we shouldn't really infer
             value_shape = infer_shape(space, w_value)
             value_size = size_from_shape(value_shape)
-
         except OperationError, e:
             if e.match(space, space.w_TypeError):
-                value_size = 1
-                value_shape = [1]
+                value_shape = []
+                value_size = 0
             else: raise
 
-        if squeeze_shape(value_shape) != squeeze_shape(shape):
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("shape mismatch: objects cannot"
-                                            " be broadcast to a single shape"))
+        if size == 0:
+            if len(value_shape) > 0:
+                raise OperationError(space.w_ValueError,
+                                     space.wrap("shape mismatch: objects cannot"
+                                                " be broadcast to a single shape"))
 
-        if size == 1:
-            self.setitem(space,
-                         self.flatten_index(slice_starts),
-                         w_value)
+            self.setitem(space, offset, self.dtype.dtype.coerce(space, w_value))
         else:
-            if value_size == 1:
-                self.set_slice_single_value(space,
-                                            slice_starts, shape, slice_steps,
-                                            w_value)
-            else:
-                self.set_slice(space,
-                               slice_starts, shape, slice_steps,
-                               w_value)
+            if squeeze_shape(value_shape) != squeeze_shape(shape):
+                raise OperationError(space.w_ValueError,
+                                     space.wrap("shape mismatch: objects cannot"
+                                                " be broadcast to a single shape"))
+
+            self.set_slice(space,
+                           offset, shape, slice_steps, strides,
+                           w_value)
     descr_setitem.unwrap_spec = ['self', ObjSpace, W_Root, W_Root]
 
     def descr_repr(self, space):
     return space.wrap(self.dtype)
 
 def descr_get_shape(space, self):
-    return space.newtuple([space.wrap(x) for x in self.shape[self.prefix:]])
+    return space.newtuple([space.wrap(x) for x in self.shape])
 
 def descr_get_array_interface(space, self):
     w_dict = space.newdict()

pypy/module/micronumpy/test/test_numpy.py

             assert typecode == infer_from_iterable(space, w_xs).typecode
 
 class TestMicroArray(object):
+    @py.test.mark.xfail # XXX: return types changed
     def test_index2strides(self, space):
         from pypy.module.micronumpy.microarray import MicroArray
         from pypy.module.micronumpy.dtype import w_int_descr
 
         offset = squeeze(slice_starts, shape, slice_steps, strides)
 
+    @py.test.mark.xfail # XXX: arguments changed
     def test_slice_setting(self, space):
         from pypy.module.micronumpy.array import size_from_shape
         from pypy.module.micronumpy.microarray import MicroArray
         assert row_strides(shape) == [30, 6, 2, 1]
         assert column_strides(shape) == [1, 7, 35, 105]
 
-    def test_flatten_index(self, space):
-        from pypy.module.micronumpy.microarray import MicroArray
-        from pypy.module.micronumpy.dtype import w_int_descr
-
-        ar = MicroArray(shape=[7, 5, 3, 2],
-                        dtype=w_int_descr)
-        
-        offset = ar.flatten_index([0, 0, 0, 0])
-        assert offset == 0
-
-        offset = ar.flatten_index([0, 0, 0, 1])
-        assert offset == 1
-
-        offset = ar.flatten_index([0, 0, 1, 1])
-        assert offset == 3
-
-        offset = ar.flatten_index([0, 2, 0, 1])
-        assert offset == 13
-
     def test_memory_layout(self, space):
         from pypy.module.micronumpy.microarray import MicroArray
         from pypy.module.micronumpy.microarray import array
         memlen = len(column_major)
 
         ar = array(space, w_data, w_dtype=w_int_descr, order='C') #C for C not column
+
         for i in range(memlen):
             array_element = space.unwrap(ar.getitem(space, i)) # ugly, but encapsulates everything
-            assert array_element == row_major[i], "Array Data: %r, Array Index: %d (%s != %s)" % (ar.dtype.dtype.dump(ar.data), i, array_element, row_major[i])
+            assert array_element == row_major[i], "Array Data: %r, Array Index: %d (%s != %s)" % (ar.dtype.dtype.dump(ar.data, 6), i, array_element, row_major[i])
 
         ar = array(space, w_data, w_dtype=w_int_descr, order='F')
         for i in range(memlen):
             array_element = space.unwrap(ar.getitem(space, i)) # ugly, but encapsulates everything
-            assert array_element == column_major[i], "Array Data: %r, Array Index: %d (%s != %s)" % (ar.dtype.dtype.dump(ar.data), i, array_element, row_major[i])
+            assert array_element == column_major[i], "Array Data: %r, Array Index: %d (%s != %s)" % (ar.dtype.dtype.dump(ar.data, 6), i, array_element, row_major[i])

pypy/module/posix/interp_posix.py

 from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2
 from pypy.rpython.module.ll_os import RegisterOs
 from pypy.rpython.module import ll_os_stat
-from pypy.rpython.lltypesystem import rffi, lltype
-from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
 import os, sys
 
 
 class State:
+    from pypy.rpython.lltypesystem import rffi, lltype
     def __init__(self, space): 
         self.space = space
         self.w_environ = space.newdict()
 
 if _WIN:
     from pypy.rlib import rwin32
+    from pypy.rpython.lltypesystem import rffi, lltype
+    from pypy.rpython.tool import rffi_platform
 
     eci = ExternalCompilationInfo(
         includes = ['windows.h', 'wincrypt.h'],