Commits

Armin Rigo committed 374c311 Merge

merge heads

Comments (0)

Files changed (11)

pypy/doc/whatsnew-head.rst

 .. branch: less-stringly-ops
 Use subclasses of SpaceOperation instead of SpaceOperator objects.
 Random cleanups in flowspace and annotator.
+
+.. branch: ndarray-buffer
+adds support for the buffer= argument to the ndarray ctor

pypy/interpreter/buffer.py

     def get_raw_address(self):
         raise ValueError("no raw buffer")
 
+    def is_writable(self):
+        return False
+
     # __________ app-level support __________
 
     def descr_len(self, space):
 
     __slots__ = ()     # no extra slot here
 
+    def is_writable(self):
+        return True
+
     def setitem(self, index, char):
         "Write a character into the buffer."
         raise NotImplementedError   # Must be overriden.  No bounds checks.

pypy/module/micronumpy/app_numpy.py

     if dtype is None:
         test = _numpypy.multiarray.array([start, stop, step, 0])
         dtype = test.dtype
-    arr = _numpypy.multiarray.zeros(int(math.ceil((stop - start) / step)), dtype=dtype)
+    length = math.ceil((float(stop) - start) / step)
+    length = int(length)
+    arr = _numpypy.multiarray.zeros(length, dtype=dtype)
     i = start
     for j in range(arr.size):
         arr[j] = i

pypy/module/micronumpy/arrayimpl/concrete.py

     def __del__(self):
         free_raw_storage(self.storage, track_allocation=False)
 
+class ConcreteArrayWithBase(ConcreteArrayNotOwning):
+    def __init__(self, shape, dtype, order, strides, backstrides, storage, orig_base):
+        ConcreteArrayNotOwning.__init__(self, shape, dtype, order,
+                                        strides, backstrides, storage)
+        self.orig_base = orig_base
+
+    def base(self):
+        return self.orig_base
+
+
+class ConcreteNonWritableArrayWithBase(ConcreteArrayWithBase):
+    def descr_setitem(self, space, orig_array, w_index, w_value):
+        raise OperationError(space.w_ValueError, space.wrap(
+            "assignment destination is read-only"))
+
 
 class NonWritableArray(ConcreteArray):
     def descr_setitem(self, space, orig_array, w_index, w_value):

pypy/module/micronumpy/base.py

         return W_NDimArray(impl)
 
     @staticmethod
-    def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False, w_subtype=None):
+    def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False,
+                               w_subtype=None, w_base=None, writable=True):
         from pypy.module.micronumpy.arrayimpl import concrete
         assert shape
         strides, backstrides = calc_strides(shape, dtype, order)
-        if owning:
+        if w_base is not None:
+            if owning:
+                raise OperationError(space.w_ValueError, 
+                        space.wrap("Cannot have owning=True when specifying a buffer"))
+            if writable:
+                impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides,
+                                                      backstrides, storage, w_base)
+            else:
+                impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype, order,
+                                                                 strides, backstrides,
+                                                                 storage, w_base)
+
+        elif owning:
             # Will free storage when GCd
             impl = concrete.ConcreteArray(shape, dtype, order, strides,
                                                 backstrides, storage=storage)

pypy/module/micronumpy/interp_numarray.py

+from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rawstorage import RAW_STORAGE_PTR
 from pypy.interpreter.error import operationerrfmt, OperationError
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr
 from pypy.interpreter.gateway import interp2app, unwrap_spec, applevel, \
 from rpython.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.arrayimpl.base import BaseArrayImplementation
 from pypy.module.micronumpy.conversion_utils import order_converter, multi_axis_converter
+from pypy.module.micronumpy import support
 from pypy.module.micronumpy.constants import *
 
 def _find_shape(space, w_size, dtype):
                     offset=0, w_strides=None, order='C'):
     from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArray
     from pypy.module.micronumpy.support import calc_strides
-    if (offset != 0 or not space.is_none(w_strides) or
-        not space.is_none(w_buffer)):
-        raise OperationError(space.w_NotImplementedError,
-                             space.wrap("unsupported param"))
     dtype = space.interp_w(interp_dtype.W_Dtype,
           space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype))
     shape = _find_shape(space, w_shape, dtype)
+
+    if not space.is_none(w_buffer):
+        if (not space.is_none(w_strides)):
+            raise OperationError(space.w_NotImplementedError,
+                                 space.wrap("unsupported param"))
+
+        buf = space.buffer_w(w_buffer)
+        try:
+            raw_ptr = buf.get_raw_address()
+        except ValueError:
+            raise OperationError(space.w_TypeError, space.wrap(
+                "Only raw buffers are supported"))
+        if not shape:
+            raise OperationError(space.w_TypeError, space.wrap(
+                "numpy scalars from buffers not supported yet"))
+        totalsize = support.product(shape) * dtype.get_size()
+        if totalsize+offset > buf.getlength():
+            raise OperationError(space.w_TypeError, space.wrap(
+                "buffer is too small for requested array"))
+        storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr)
+        storage = rffi.ptradd(storage, offset)
+        return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
+                                                  w_subtype=w_subtype,
+                                                  w_base=w_buffer,
+                                                  writable=buf.is_writable())
+
     if not shape:
         return W_NDimArray.new_scalar(space, dtype)
     if space.is_w(w_subtype, space.gettypefor(W_NDimArray)):
     Create an array from an existing buffer, given its address as int.
     PyPy-only implementation detail.
     """
-    from rpython.rtyper.lltypesystem import rffi
-    from rpython.rlib.rawstorage import RAW_STORAGE_PTR
     storage = rffi.cast(RAW_STORAGE_PTR, addr)
     dtype = space.interp_w(interp_dtype.W_Dtype,
                      space.call_function(space.gettypefor(interp_dtype.W_Dtype),

pypy/module/micronumpy/test/test_numarray.py

 
 class AppTestNumArray(BaseNumpyAppTest):
     spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
+
     def w_CustomIndexObject(self, index):
         class CustomIndexObject(object):
             def __init__(self, index):
         assert a.dtype is dtype(int)
         a = arange(3, 7, 2)
         assert (a == [3, 5]).all()
+        a = arange(3, 8, 2)
+        assert (a == [3, 5, 7]).all()
         a = arange(3, dtype=float)
         assert (a == [0., 1., 2.]).all()
         assert a.dtype is dtype(float)
         a = np.ndarray([1], dtype=bool)
         assert a[0] == True
 
+
+class AppTestNumArrayFromBuffer(BaseNumpyAppTest):
+    spaceconfig = dict(usemodules=["micronumpy", "array", "mmap"])
+
+    def setup_class(cls):
+        from rpython.tool.udir import udir
+        BaseNumpyAppTest.setup_class.im_func(cls)
+        cls.w_tmpname = cls.space.wrap(str(udir.join('mmap-')))
+
+    def test_ndarray_from_buffer(self):
+        import numpypy as np
+        import array
+        buf = array.array('c', ['\x00']*2*3)
+        a = np.ndarray((3,), buffer=buf, dtype='i2')
+        a[0] = ord('b')
+        a[1] = ord('a')
+        a[2] = ord('r')
+        assert list(buf) == ['b', '\x00', 'a', '\x00', 'r', '\x00']
+        assert a.base is buf
+
+    def test_ndarray_subclass_from_buffer(self):
+        import numpypy as np
+        import array
+        buf = array.array('c', ['\x00']*2*3)
+        class X(np.ndarray):
+            pass
+        a = X((3,), buffer=buf, dtype='i2')
+        assert type(a) is X
+
+    def test_ndarray_from_buffer_and_offset(self):
+        import numpypy as np
+        import array
+        buf = array.array('c', ['\x00']*7)
+        buf[0] = 'X'
+        a = np.ndarray((3,), buffer=buf, offset=1, dtype='i2')
+        a[0] = ord('b')
+        a[1] = ord('a')
+        a[2] = ord('r')
+        assert list(buf) == ['X', 'b', '\x00', 'a', '\x00', 'r', '\x00']
+
+    def test_ndarray_from_buffer_out_of_bounds(self):
+        import numpypy as np
+        import array
+        buf = array.array('c', ['\x00']*2*10) # 20 bytes
+        info = raises(TypeError, "np.ndarray((11,), buffer=buf, dtype='i2')")
+        assert str(info.value).startswith('buffer is too small')
+        info = raises(TypeError, "np.ndarray((5,), buffer=buf, offset=15, dtype='i2')")
+        assert str(info.value).startswith('buffer is too small')
+
+    def test_ndarray_from_readonly_buffer(self):
+        import numpypy as np
+        from mmap import mmap, ACCESS_READ
+        f = open(self.tmpname, "w+")
+        f.write("hello")
+        f.flush()
+        buf = mmap(f.fileno(), 5, access=ACCESS_READ)
+        a = np.ndarray((5,), buffer=buf, dtype='c')
+        raises(ValueError, "a[0] = 'X'")
+        buf.close()
+        f.close()
+
+
+
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import numpypy

pypy/module/mmap/interp_mmap.py

         self.check_valid_writeable()
         self.mmap.setslice(start, string)
 
+    def is_writable(self):
+        try:
+            self.mmap.check_writeable()
+        except RMMapError:
+            return False
+        else:
+            return True
+
     def get_raw_address(self):
         self.check_valid()
         return self.mmap.data

pypy/module/pypyjit/interp_jit.py

 """This is not the JIT :-)
 
-This is transformed to become a JIT by code elsewhere: pypy/jit/*
+This is transformed to become a JIT by code elsewhere: rpython/jit/*
 """
 
-from rpython.tool.pairtype import extendabletype
 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside
 from rpython.rlib import jit
 from rpython.rlib.jit import current_trace_length, unroll_parameters
 import pypy.interpreter.pyopcode   # for side-effects
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.pycode import PyCode, CO_GENERATOR
+from pypy.interpreter.pycode import CO_GENERATOR
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pyopcode import ExitFrame, Yield
 from opcode import opmap
 
+
 PyFrame._virtualizable_ = ['last_instr', 'pycode',
                            'valuestackdepth', 'locals_stack_w[*]',
                            'cells[*]',

pypy/testrunner_cfg.py

 
 DIRS_SPLIT = [
     'translator/c', 'rlib',
-    'rpython/memory', 'jit/metainterp', 'rpython/test',
+    'memory/test', 'jit/metainterp',
     'jit/backend/arm', 'jit/backend/x86',
 ]
 

rpython/rlib/rfile.py

 
 eci = ExternalCompilationInfo(includes=['stdio.h', 'unistd.h', 'sys/types.h'])
 
-def llexternal(*args):
-    return rffi.llexternal(*args, compilation_info=eci)
+def llexternal(*args, **kwargs):
+    return rffi.llexternal(*args, compilation_info=eci, **kwargs)
 
 FILE = lltype.Struct('FILE') # opaque type maybe
 
 
     off_t = platform.SimpleType('off_t')
 
+
 CC = platform.configure(CConfig)
 OFF_T = CC['off_t']
-
 c_open = llexternal('fopen', [rffi.CCHARP, rffi.CCHARP], lltype.Ptr(FILE))
 c_close = llexternal('fclose', [lltype.Ptr(FILE)], rffi.INT)
 c_write = llexternal('fwrite', [rffi.CCHARP, rffi.SIZE_T, rffi.SIZE_T,
 c_fileno = llexternal('fileno', [lltype.Ptr(FILE)], rffi.INT)
 c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], lltype.Signed)
 c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
-c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT)
+c_ftruncate = llexternal('ftruncate', [rffi.INT, OFF_T], rffi.INT, macro=True)
+
 c_fgets = llexternal('fgets', [rffi.CCHARP, rffi.INT, lltype.Ptr(FILE)],
                      rffi.CCHARP)