Alex Gaynor avatar Alex Gaynor committed 8090d3a Merge

merged upstream

Comments (0)

Files changed (15)

lib-python/2/collections.py

 import heapq as _heapq
 from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
 from itertools import imap as _imap
+try:
+    from __pypy__ import newdict
+except ImportError:
+    newdict = lambda _ : {}
 
 try:
     from thread import get_ident as _get_ident
 
     # Execute the template string in a temporary namespace and
     # support tracing utilities by setting a value for frame.f_globals['__name__']
-    namespace = dict(__name__='namedtuple_%s' % typename,
-                     OrderedDict=OrderedDict, _property=property, _tuple=tuple)
+    namespace = newdict('module')
+    namespace['OrderedDict'] = OrderedDict
+    namespace['_property'] = property
+    namespace['_tuple'] = tuple
+    namespace['__name__'] = 'namedtuple_%s' % typename
     try:
         exec template in namespace
     except SyntaxError, e:

pypy/doc/whatsnew-head.rst

 
 .. branch: numpypy-longdouble
 Long double support for numpypy
+
 .. branch: numpypy-disable-longdouble
 Since r_longdouble support is missing, disable all longdouble and derivative
 dtypes using ENABLED_LONG_DOUBLE = False
+
 .. branch: numpypy-real-as-view
 Convert real, imag from ufuncs to views. This involves the beginning of
 view() functionality
+
 .. branch: indexing-by-array
 Adds indexing by scalar, adds int conversion from scalar and single element array,
 fixes compress, indexing by an array with a smaller shape and the indexed object.
 
+.. branch: str-dtype-improvement
+Allow concatenation of str and numeric arrays
+
 .. branch: signatures
 Improved RPython typing
 

pypy/module/micronumpy/arrayimpl/concrete.py

             return
         shape = shape_agreement(space, self.get_shape(), arr)
         if impl.storage == self.storage:
-            impl = impl.copy()
-        loop.setslice(shape, self, impl)
+            impl = impl.copy(space)
+        loop.setslice(space, shape, self, impl)
 
     def get_size(self):
         return self.size // self.dtype.itemtype.get_element_size()
         return SliceArray(self.start, strides,
                           backstrides, shape, self, orig_array)
 
-    def copy(self):
+    def copy(self, space):
         strides, backstrides = support.calc_strides(self.get_shape(), self.dtype,
                                                     self.order)
         impl = ConcreteArray(self.get_shape(), self.dtype, self.order, strides,
                              backstrides)
-        return loop.setslice(self.get_shape(), impl, self)
+        return loop.setslice(space, self.get_shape(), impl, self)
 
     def create_axis_iter(self, shape, dim, cum):
         return iter.AxisIterator(self, shape, dim, cum)
 
     def astype(self, space, dtype):
         new_arr = W_NDimArray.from_shape(self.get_shape(), dtype)
-        loop.copy_from_to(self, new_arr.implementation, dtype)
+        if dtype.is_str_or_unicode():
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "astype(%s) not implemented yet" % self.dtype))
+        else:    
+            loop.setslice(space, new_arr.get_shape(), new_arr.implementation, self)
         return new_arr
 
 class ConcreteArrayNotOwning(BaseConcreteArray):

pypy/module/micronumpy/arrayimpl/scalar.py

     def set_scalar_value(self, w_val):
         self.value = w_val.convert_to(self.dtype)
 
-    def copy(self):
+    def copy(self, space):
         scalar = Scalar(self.dtype)
         scalar.value = self.value
         return scalar
                 "(%s) into shape ()" % (
                     ','.join([str(x) for x in w_arr.get_shape()],))))
         if self.dtype.is_complex_type():
-            #imag = dtype.itemtype.unbox(self.value.convert_imag_to(dtype))
-            #val = dtype.itemtype.unbox(w_arr.get_scalar_value().
-            #                                           convert_to(dtype))
-            #self.value = self.dtype.box_complex(val, imag)
-            self.value = self.dtype.itemtype.composite(w_arr.get_scalar_value().convert_to(dtype),
-                                    self.value.convert_imag_to(dtype))
+            self.value = self.dtype.itemtype.composite(
+                               w_arr.get_scalar_value().convert_to(dtype),
+                               self.value.convert_imag_to(dtype))
         else:
             self.value = w_arr.get_scalar_value()
 
                 "could not broadcast input array from shape " + 
                 "(%s) into shape ()" % (
                     ','.join([str(x) for x in w_arr.get_shape()],))))
-        #real = dtype.itemtype.unbox(self.value.convert_real_to(dtype))
-        #val = dtype.itemtype.unbox(w_arr.get_scalar_value().
-        #                                              convert_to(dtype))
-        #self.value = self.dtype.box_complex(real, val)
         self.value = self.dtype.itemtype.composite(
                             self.value.convert_real_to(dtype),
-                            w_arr.get_scalar_value(),
+                            w_arr.get_scalar_value().convert_to(dtype),
                             )
 
     def descr_getitem(self, space, _, w_idx):

pypy/module/micronumpy/interp_arrayops.py

                     "all the input arrays must have same number of dimensions"))
             elif i == _axis:
                 shape[i] += axis_size
+        a_dt = arr.get_dtype()
+        if dtype.is_record_type() and a_dt.is_record_type():
+            #Record types must match
+            for f in dtype.fields:
+                if f not in a_dt.fields or \
+                             dtype.fields[f] != a_dt.fields[f]:
+                    raise OperationError(space.w_TypeError, 
+                               space.wrap("record type mismatch"))
+        elif dtype.is_record_type() or a_dt.is_record_type():
+            raise OperationError(space.w_TypeError, 
+                        space.wrap("invalid type promotion"))
         dtype = interp_ufuncs.find_binop_result_dtype(space, dtype,
                                                       arr.get_dtype())
         if _axis < 0 or len(arr.get_shape()) <= _axis:
             raise operationerrfmt(space.w_IndexError, "axis %d out of bounds [0, %d)", axis, len(shape))
-    if _axis < 0 or len(shape) <= _axis:
-        raise operationerrfmt(space.w_IndexError, "axis %d out of bounds [0, %d)", axis, len(shape))
     res = W_NDimArray.from_shape(shape, dtype, 'C')
     chunks = [Chunk(0, i, 1, i) for i in shape]
     axis_start = 0

pypy/module/micronumpy/interp_boxes.py

         dtype.itemtype.store(self.arr, self.ofs, ofs,
                              dtype.coerce(space, w_value))
 
+    def convert_to(self, dtype):
+        # if we reach here, the record fields are guarenteed to match.
+        return self
+
 class W_CharacterBox(W_FlexibleBox):
     pass
 
             arr.storage[i] = arg[i]
         return W_StringBox(arr, 0, arr.dtype)
 
-    def convert_to(self, dtype):
-        from pypy.module.micronumpy import types
-        assert isinstance(dtype.itemtype, types.StringType)
-        return self
 
 class W_UnicodeBox(W_CharacterBox):
     def descr__new__unicode_box(space, w_subtype, w_arg):
         #    arr.storage[i] = arg[i]
         return W_UnicodeBox(arr, 0, arr.dtype)
 
-    def convert_to(self, dtype):
-        from pypy.module.micronumpy import types
-        assert isinstance(dtype.itemtype, types.UnicodeType)
-        return self
-
 
 class W_ComplexFloatingBox(W_InexactBox):
     _attrs_ = ()

pypy/module/micronumpy/interp_dtype.py

     def box_complex(self, real, imag):
         return self.itemtype.box_complex(real, imag)
 
+    def build_and_convert(self, space, box):
+        return self.itemtype.build_and_convert(space, self, box)
     def coerce(self, space, w_item):
         return self.itemtype.coerce(space, self, w_item)
 

pypy/module/micronumpy/interp_flatiter.py

         base = self.base
         start, stop, step, length = space.decode_index4(w_idx, base.get_size())
         arr = convert_to_array(space, w_value)
-        loop.flatiter_setitem(self.base, arr, start, step, length)
+        loop.flatiter_setitem(space, self.base, arr, start, step, length)
 
     def descr_iter(self):
         return self

pypy/module/micronumpy/interp_numarray.py

         return self.implementation.get_scalar_value()
 
     def descr_copy(self, space):
-        return W_NDimArray(self.implementation.copy())
+        return W_NDimArray(self.implementation.copy(space))
 
     def descr_get_real(self, space):
         return W_NDimArray(self.implementation.get_real(self))
 
     def descr_get_imag(self, space):
         ret = self.implementation.get_imag(self)
-        if ret:
-            return W_NDimArray(ret)
-        raise OperationError(space.w_NotImplementedError,
-                    space.wrap('imag not implemented for this dtype'))
+        return W_NDimArray(ret)
 
     def descr_set_real(self, space, w_value):
         # copy (broadcast) values into self

pypy/module/micronumpy/interp_ufuncs.py

     if promote_to_float:
         return find_unaryop_result_dtype(space, dt2, promote_to_float=True)
     # If they're the same kind, choose the greater one.
-    if dt1.kind == dt2.kind:
+    if dt1.kind == dt2.kind and not dt2.is_flexible_type():
         return dt2
 
     # Everything promotes to float, and bool promotes to everything.
     elif dt2.num == 10 or (LONG_BIT == 64 and dt2.num == 8):
         # UInt64 + signed = Float64
         dtypenum = 12
-    else:
+    elif dt2.is_flexible_type():
+        # For those operations that get here (concatenate, stack),
+        # flexible types take precedence over numeric type
+        if dt2.is_record_type():
+            return dt2
+        if dt1.is_str_or_unicode():
+            if dt2.num == 18:
+                if dt2.itemtype.get_element_size() >= \
+                           dt1.itemtype.get_element_size():
+                    return dt2
+                return dt1
+            if dt2.itemtype.get_element_size() >= \
+                       dt1.itemtype.get_element_size():
+                return dt2
+            return dt1
+        return dt2
+    else:    
         # increase to the next signed type
         dtypenum = dt2.num + 1
     newdtype = interp_dtype.get_dtype_cache(space).dtypes_by_num[dtypenum]

pypy/module/micronumpy/loop.py

         obj_iter.next()
     return out
 
-setslice_driver = jit.JitDriver(name='numpy_setslice',
+setslice_driver1 = jit.JitDriver(name='numpy_setslice1',
                                 greens = ['shapelen', 'dtype'],
-                                reds = ['target', 'source', 'target_iter',
-                                        'source_iter'])
+                                reds = 'auto')
+setslice_driver2 = jit.JitDriver(name='numpy_setslice2',
+                                greens = ['shapelen', 'dtype'],
+                                reds = 'auto')
 
-def setslice(shape, target, source):
+def setslice(space, shape, target, source):
+    if target.dtype.is_str_or_unicode():
+        return setslice_build_and_convert(space, shape, target, source)
+    return setslice_to(space, shape, target, source)
+
+def setslice_to(space, shape, target, source):
     # note that unlike everything else, target and source here are
     # array implementations, not arrays
     target_iter = target.create_iter(shape)
     dtype = target.dtype
     shapelen = len(shape)
     while not target_iter.done():
-        setslice_driver.jit_merge_point(shapelen=shapelen, dtype=dtype,
-                                        target=target, source=source,
-                                        target_iter=target_iter,
-                                        source_iter=source_iter)
+        setslice_driver1.jit_merge_point(shapelen=shapelen, dtype=dtype)
         target_iter.setitem(source_iter.getitem().convert_to(dtype))
         target_iter.next()
         source_iter.next()
     return target
 
+def setslice_build_and_convert(space, shape, target, source):
+    # note that unlike everything else, target and source here are
+    # array implementations, not arrays
+    target_iter = target.create_iter(shape)
+    source_iter = source.create_iter(shape)
+    dtype = target.dtype
+    shapelen = len(shape)
+    while not target_iter.done():
+        setslice_driver2.jit_merge_point(shapelen=shapelen, dtype=dtype)
+        target_iter.setitem(dtype.build_and_convert(space, source_iter.getitem()))
+        target_iter.next()
+        source_iter.next()
+    return target
+
 reduce_driver = jit.JitDriver(name='numpy_reduce',
                               greens = ['shapelen', 'func', 'done_func',
                                         'calc_dtype', 'identity'],
         ri.next()
     return res
 
-flatiter_setitem_driver = jit.JitDriver(name = 'numpy_flatiter_setitem',
+flatiter_setitem_driver1 = jit.JitDriver(name = 'numpy_flatiter_setitem1',
                                         greens = ['dtype'],
                                         reds = 'auto')
 
-def flatiter_setitem(arr, val, start, step, length):
+flatiter_setitem_driver2 = jit.JitDriver(name = 'numpy_flatiter_setitem2',
+                                        greens = ['dtype'],
+                                        reds = 'auto')
+
+def flatiter_setitem(space, arr, val, start, step, length):
+    dtype = arr.get_dtype()
+    if dtype.is_str_or_unicode():
+        return flatiter_setitem_build_and_convert(space, arr, val, start, step, length)
+    return flatiter_setitem_to(space, arr, val, start, step, length)
+
+def flatiter_setitem_to(space, arr, val, start, step, length):
     dtype = arr.get_dtype()
     arr_iter = arr.create_iter()
     val_iter = val.create_iter()
     arr_iter.next_skip_x(start)
     while length > 0:
-        flatiter_setitem_driver.jit_merge_point(dtype=dtype)
+        flatiter_setitem_driver1.jit_merge_point(dtype=dtype)
         arr_iter.setitem(val_iter.getitem().convert_to(dtype))
         # need to repeat i_nput values until all assignments are done
         arr_iter.next_skip_x(step)
         # WTF numpy?
         val_iter.reset()
 
+def flatiter_setitem_build_and_convert(space, arr, val, start, step, length):
+    dtype = arr.get_dtype()
+    arr_iter = arr.create_iter()
+    val_iter = val.create_iter()
+    arr_iter.next_skip_x(start)
+    while length > 0:
+        flatiter_setitem_driver2.jit_merge_point(dtype=dtype)
+        arr_iter.setitem(dtype.build_and_convert(space, val_iter.getitem()))
+        # need to repeat i_nput values until all assignments are done
+        arr_iter.next_skip_x(step)
+        length -= 1
+        val_iter.next()
+        # WTF numpy?
+        val_iter.reset()
+
 fromstring_driver = jit.JitDriver(name = 'numpy_fromstring',
                                   greens = ['itemsize', 'dtype'],
                                   reds = 'auto')
                           val_arr.descr_getitem(space, w_idx))
         iter.next()
 
-copy_from_to_driver = jit.JitDriver(greens = ['dtype'],
-                                    reds = 'auto')
-
-def copy_from_to(from_, to, dtype):
-    from_iter = from_.create_iter()
-    to_iter = to.create_iter()
-    while not from_iter.done():
-        copy_from_to_driver.jit_merge_point(dtype=dtype)
-        to_iter.setitem(from_iter.getitem().convert_to(dtype))
-        to_iter.next()
-        from_iter.next()
-
 byteswap_driver = jit.JitDriver(greens = ['dtype'],
                                     reds = 'auto')
 

pypy/module/micronumpy/test/test_complex.py

         a = array(complex(3.0, 4.0))
         b = a.real
         assert b == array(3)
+        assert a.imag == array(4)
         a.real = 1024
-        assert a.real == 1024 
-        assert a.imag == array(4)
+        a.imag = 2048
+        assert a.real == 1024 and a.imag == 2048
         assert b.dtype == dtype(float)
         a = array(4.0)
         b = a.imag
         assert b == 0
         assert b.dtype == dtype(float)
-        raises(TypeError, 'a.imag = 1024')
-        raises(ValueError, 'a.real = [1, 3]')
+        exc = raises(TypeError, 'a.imag = 1024')
+        assert str(exc.value).startswith("array does not have imaginary")
+        exc = raises(ValueError, 'a.real = [1, 3]')
+        assert str(exc.value) == \
+            "could not broadcast input array from shape (2) into shape ()"
         a = array('abc')
         assert str(a.real) == 'abc'
         # numpy imag for flexible types returns self
             assert repr(abs(complex(float('nan'), float('nan')))) == 'nan'
             # numpy actually raises an AttributeError,
             # but numpypy raises a TypeError
-            raises((TypeError, AttributeError), 'c2.real = 10.')
-            raises((TypeError, AttributeError), 'c2.imag = 10.')
+            exc = raises((TypeError, AttributeError), 'c2.real = 10.')
+            assert str(exc.value) == "readonly attribute"
+            exc = raises((TypeError, AttributeError), 'c2.imag = 10.')
+            assert str(exc.value) == "readonly attribute"
             assert(real(c2) == 3.0)
             assert(imag(c2) == 4.0)
 

pypy/module/micronumpy/test/test_numarray.py

         a = (a + a)[::2]
         b = concatenate((a[:3], a[-3:]))
         assert (b == [2, 6, 10, 2, 6, 10]).all()
+        a = concatenate((array([1]), array(['abc'])))
+        assert str(a.dtype) == '|S3'
+        a = concatenate((array([]), array(['abc'])))
+        assert a[0] == 'abc'
+        a = concatenate((['abcdef'], ['abc']))
+        assert a[0] == 'abcdef'
+        assert str(a.dtype) == '|S6'
+    
+    def test_record_concatenate(self):
+        # only an exact match can succeed
+        from numpypy import zeros, concatenate
+        a = concatenate((zeros((2,),dtype=[('x', int), ('y', float)]),
+                         zeros((2,),dtype=[('x', int), ('y', float)])))
+        assert a.shape == (4,)
+        exc = raises(TypeError, concatenate, 
+                            (zeros((2,), dtype=[('x', int), ('y', float)]),
+                            (zeros((2,), dtype=[('x', float), ('y', float)]))))
+        assert str(exc.value).startswith('record type mismatch')
+        exc = raises(TypeError, concatenate, ([1], zeros((2,),
+                                            dtype=[('x', int), ('y', float)])))
+        assert str(exc.value).startswith('invalid type promotion')
+        exc = raises(TypeError, concatenate, (['abc'], zeros((2,),
+                                            dtype=[('x', int), ('y', float)])))
+        assert str(exc.value).startswith('invalid type promotion')
+
+
 
     def test_std(self):
         from numpypy import array
 
         a = array('x').astype('S3').dtype
         assert a.itemsize == 3
+        # scalar vs. array
+        try:
+            a = array([1, 2, 3.14156]).astype('S3').dtype
+            assert a.itemsize == 3
+        except NotImplementedError:
+            skip('astype("S3") not implemented for numeric arrays')
 
     def test_base(self):
         from numpypy import array
         assert (a.transpose() == b).all()
 
     def test_flatiter(self):
-        from numpypy import array, flatiter, arange
+        from numpypy import array, flatiter, arange, zeros
         a = array([[10, 30], [40, 60]])
         f_iter = a.flat
         assert f_iter.next() == 10
         a = arange(10).reshape(5, 2)
         raises(IndexError, 'a.flat[(1, 2)]')
         assert a.flat.base is a
+        m = zeros((2,2), dtype='S3')
+        m.flat[1] = 1
+        assert m[0,1] == '1'
 
     def test_flatiter_array_conv(self):
         from numpypy import array, dot

pypy/module/micronumpy/types.py

     def get_size(self):
         return self.size
 
+
 class StringType(BaseType, BaseStringType):
     T = lltype.Char
 
     def coerce(self, space, dtype, w_item):
         from pypy.module.micronumpy.interp_dtype import new_string_dtype
         arg = space.str_w(space.str(w_item))
-        arr = interp_boxes.VoidBoxStorage(len(arg), new_string_dtype(space, len(arg)))
+        arr = VoidBoxStorage(len(arg), new_string_dtype(space, len(arg)))
         for i in range(len(arg)):
             arr.storage[i] = arg[i]
         return interp_boxes.W_StringBox(arr,  0, arr.dtype)
     def to_builtin_type(self, space, box):
         return space.wrap(self.to_str(box))
 
+    def build_and_convert(self, space, mydtype, box):
+        if box.get_dtype(space).is_str_or_unicode():
+            arg = box.get_dtype(space).itemtype.to_str(box)
+        else:
+            w_arg = box.descr_str(space)
+            arg = space.str_w(space.str(w_arg))
+        arr = VoidBoxStorage(self.size, mydtype)
+        i = 0
+        for i in range(min(len(arg), self.size)):
+            arr.storage[i] = arg[i]
+        for j in range(i + 1, self.size):
+            arr.storage[j] = '\x00'
+        return interp_boxes.W_StringBox(arr,  0, arr.dtype)
+        
 class VoidType(BaseType, BaseStringType):
     T = lltype.Char
 

pypy/objspace/std/dictmultiobject.py

         for w_k, w_v in list_pairs_w:
             w_self.setitem(w_k, w_v)
 
-    def setitem(self, w_key, w_value):
-        self.strategy.setitem(self, w_key, w_value)
-
     def setitem_str(self, key, w_value):
         self.strategy.setitem_str(self, key, w_value)
 
 
 def _add_indirections():
-    dict_methods = "getitem \
-                    getitem_str delitem length \
-                    clear w_keys values \
-                    items iterkeys itervalues iteritems setdefault \
-                    popitem listview_str listview_unicode listview_int \
+    dict_methods = "getitem getitem_str setitem setdefault \
+                    popitem delitem clear \
+                    length w_keys values items \
+                    iterkeys itervalues iteritems \
+                    listview_str listview_unicode listview_int \
                     view_as_kwargs".split()
 
     def make_method(method):
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.