Commits

Alex Gaynor committed 607d852 Merge

merged default in?

Comments (0)

Files changed (62)

lib_pypy/numpypy/core/numeric.py

 import _numpypy as multiarray # ARGH
 from numpypy.core.arrayprint import array2string
 
-
+newaxis = None
 
 def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
     """
 False_ = bool_(False)
 True_ = bool_(True)
 e = math.e
-pi = math.pi
+pi = math.pi
 # Empty
+
+# XXX Should be empty again, soon.
+# XXX hack for win64:
+# This patch must stay here until the END OF STAGE 1
+# When all tests work, this branch will be merged
+# and the branch stage 2 is started, where we remove this patch.
+import sys
+if hasattr(sys, "maxsize"):
+    if sys.maxint != sys.maxsize:
+        sys.maxint = sys.maxsize
+        import warnings
+        warnings.warn("""\n
+---> This win64 port is now in stage 1: sys.maxint was modified.
+---> When pypy/__init__.py becomes empty again, we have reached stage 2.
+""")

pypy/annotation/classdef.py

                 "the attribute here; the list of read locations is:\n" +
                 '\n'.join([str(loc[0]) for loc in self.read_locations]))
 
-
 class ClassDef(object):
     "Wraps a user class."
 

pypy/interpreter/baseobjspace.py

         if not self.is_true(self.isinstance(w_obj, self.w_str)):
             raise OperationError(self.w_TypeError,
                                  self.wrap('argument must be a string'))
-        return self.str_w(w_obj)
+        return self.str_w(w_obj)            
 
     def unicode_w(self, w_obj):
         return w_obj.unicode_w(self)

pypy/interpreter/error.py

 
     def async(self, space):
         "Check if this is an exception that should better not be caught."
+        if not space.full_exceptions:
+            # flow objspace does not support such exceptions and more
+            # importantly, raises KeyboardInterrupt if you try to access
+            # space.w_KeyboardInterrupt
+            return False
         return (self.match(space, space.w_SystemExit) or
                 self.match(space, space.w_KeyboardInterrupt))
 

pypy/interpreter/gateway.py

 
     def __init__(self, source, filename=None, modname='__builtin__'):
         # HAAACK (but a good one)
+        self.filename = filename
+        self.source = str(py.code.Source(source).deindent())
+        self.modname = modname
         if filename is None:
             f = sys._getframe(1)
             filename = '<%s:%d>' % (f.f_code.co_filename, f.f_lineno)
+        if not os.path.exists(filename):
+            # make source code available for tracebacks
+            lines = [x + "\n" for x in source.split("\n")]
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
         self.filename = filename
-        self.source = str(py.code.Source(source).deindent())
-        self.modname = modname
-        # make source code available for tracebacks
-        lines = [x + "\n" for x in source.split("\n")]
-        py.std.linecache.cache[filename] = (1, None, lines, filename)
 
     def __repr__(self):
         return "<ApplevelClass filename=%r>" % (self.filename,)

pypy/jit/backend/llsupport/regalloc.py

         except KeyError:
             pass   # 'var' is already not in a register
 
-    def loc(self, box):
+    def loc(self, box, must_exist=False):
         """ Return the location of 'box'.
         """
         self._check_type(box)
         except KeyError:
             if box in self.bindings_to_frame_reg:
                 return self.frame_reg
+            if must_exist:
+                return self.frame_manager.bindings[box]
             return self.frame_manager.loc(box)
 
     def return_constant(self, v, forbidden_vars=[], selected_reg=None):
         self._check_type(v)
         if isinstance(v, Const):
             return self.return_constant(v, forbidden_vars, selected_reg)
-        prev_loc = self.loc(v)
+        prev_loc = self.loc(v, must_exist=True)
         if prev_loc is self.frame_reg and selected_reg is None:
             return prev_loc
         loc = self.force_allocate_reg(v, forbidden_vars, selected_reg,

pypy/jit/backend/llsupport/test/test_regalloc.py

-
+import py
 from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT
 from pypy.jit.backend.llsupport.regalloc import FrameManager
 from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan
         assert isinstance(loc, FakeFramePos)
         assert len(asm.moves) == 1
 
+    def test_bogus_make_sure_var_in_reg(self):
+        b0, = newboxes(0)
+        longevity = {b0: (0, 1)}
+        fm = TFrameManager()
+        asm = MockAsm()
+        rm = RegisterManager(longevity, frame_manager=fm, assembler=asm)
+        rm.next_instruction()
+        # invalid call to make_sure_var_in_reg(): box unknown so far
+        py.test.raises(KeyError, rm.make_sure_var_in_reg, b0)
+
     def test_return_constant(self):
         asm = MockAsm()
         boxes, longevity = boxes_and_longevity(5)

pypy/jit/backend/test/runner_test.py

                                                c_nest, c_nest], 'void')
 
     def test_read_timestamp(self):
+        if sys.platform == 'win32':
+            # windows quite often is very inexact (like the old Intel 8259 PIC),
+            # so we stretch the time a little bit.
+            # On my virtual Parallels machine in a 2GHz Core i7 Mac Mini,
+            # the test starts working at delay == 21670 and stops at 20600000.
+            # We take the geometric mean value.
+            from math import log, exp
+            delay_min = 21670
+            delay_max = 20600000
+            delay = int(exp((log(delay_min)+log(delay_max))/2))
+            def wait_a_bit():
+                for i in xrange(delay): pass
+        else:
+            def wait_a_bit():
+                pass
         if longlong.is_64_bit:
             got1 = self.execute_operation(rop.READ_TIMESTAMP, [], 'int')
+            wait_a_bit()
             got2 = self.execute_operation(rop.READ_TIMESTAMP, [], 'int')
             res1 = got1.getint()
             res2 = got2.getint()
         else:
             got1 = self.execute_operation(rop.READ_TIMESTAMP, [], 'float')
+            wait_a_bit()
             got2 = self.execute_operation(rop.READ_TIMESTAMP, [], 'float')
             res1 = got1.getlonglong()
             res2 = got2.getlonglong()

pypy/jit/backend/x86/assembler.py

             self.mc.MOVSD_sx(0, loc.value)
         elif WORD == 4 and isinstance(loc, StackLoc) and loc.get_width() == 8:
             # XXX evil trick
-            self.mc.PUSH_b(get_ebp_ofs(loc.position))
-            self.mc.PUSH_b(get_ebp_ofs(loc.position + 1))
+            self.mc.PUSH_b(loc.value + 4)
+            self.mc.PUSH_b(loc.value)
         else:
             self.mc.PUSH(loc)
 
             self.mc.ADD_ri(esp.value, 8)   # = size of doubles
         elif WORD == 4 and isinstance(loc, StackLoc) and loc.get_width() == 8:
             # XXX evil trick
-            self.mc.POP_b(get_ebp_ofs(loc.position + 1))
-            self.mc.POP_b(get_ebp_ofs(loc.position))
+            self.mc.POP_b(loc.value)
+            self.mc.POP_b(loc.value + 4)
         else:
             self.mc.POP(loc)
 
             mc.PUSH_r(ebx.value)
         elif IS_X86_64:
             mc.MOV_rr(edi.value, ebx.value)
-            # XXX: Correct to only align the stack on 64-bit?
-            mc.AND_ri(esp.value, -16)
         else:
             raise AssertionError("Shouldn't happen")
 
         # First, we need to save away the registers listed in
         # 'save_registers' that are not callee-save.  XXX We assume that
         # the XMM registers won't be modified.  We store them in
-        # [ESP+4], [ESP+8], etc., leaving enough room in [ESP] for the
-        # single argument to closestack_addr below.
-        p = WORD
+        # [ESP+4], [ESP+8], etc.; on x86-32 we leave enough room in [ESP]
+        # for the single argument to closestack_addr below.
+        if IS_X86_32:
+            p = WORD
+        elif IS_X86_64:
+            p = 0
         for reg in self._regalloc.rm.save_around_call_regs:
             if reg in save_registers:
                 self.mc.MOV_sr(p, reg.value)
         #
         self._emit_call(-1, imm(self.releasegil_addr), args)
         # Finally, restore the registers saved above.
-        p = WORD
+        if IS_X86_32:
+            p = WORD
+        elif IS_X86_64:
+            p = 0
         for reg in self._regalloc.rm.save_around_call_regs:
             if reg in save_registers:
                 self.mc.MOV_rs(reg.value, p)

pypy/jit/backend/x86/support.py

 
 # ____________________________________________________________
 
-if sys.platform == 'win32':
-    ensure_sse2_floats = lambda : None
-    # XXX check for SSE2 on win32 too
+if WORD == 4:
+    extra = ['-DPYPY_X86_CHECK_SSE2']
 else:
-    if WORD == 4:
-        extra = ['-DPYPY_X86_CHECK_SSE2']
-    else:
-        extra = []
-    ensure_sse2_floats = rffi.llexternal_use_eci(ExternalCompilationInfo(
-        compile_extra = ['-msse2', '-mfpmath=sse',
-                         '-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra,
-        ))
+    extra = []
+
+if sys.platform != 'win32':
+    extra = ['-msse2', '-mfpmath=sse',
+             '-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra
+
+ensure_sse2_floats = rffi.llexternal_use_eci(ExternalCompilationInfo(
+    compile_extra = extra,
+))

pypy/jit/backend/x86/test/test_zmath.py

 from pypy.translator.c.test.test_genc import compile
 from pypy.jit.backend.x86.support import ensure_sse2_floats
 from pypy.rlib import rfloat
+from pypy.rlib.unroll import unrolling_iterable
+from pypy.rlib.debug import debug_print
 
 
 def get_test_case((fnname, args, expected)):
     expect_valueerror = (expected == ValueError)
     expect_overflowerror = (expected == OverflowError)
     check = test_direct.get_tester(expected)
+    unroll_args = unrolling_iterable(args)
     #
     def testfn():
+        debug_print('calling', fnname, 'with arguments:')
+        for arg in unroll_args:
+            debug_print('\t', arg)
         try:
             got = fn(*args)
         except ValueError:
-            return expect_valueerror
+            if expect_valueerror:
+                return True
+            else:
+                debug_print('unexpected ValueError!')
+                return False
         except OverflowError:
-            return expect_overflowerror
+            if expect_overflowerror:
+                return True
+            else:
+                debug_print('unexpected OverflowError!')
+                return False
         else:
-            return check(got)
+            if check(got):
+                return True
+            else:
+                debug_print('unexpected result:', got)
+                return False
     #
     testfn.func_name = 'test_' + fnname
     return testfn

pypy/module/_ssl/test/test_ssl.py

 from pypy.conftest import gettestobjspace
 import os
 import py
-from pypy.rlib.rarithmetic import is_valid_int
 
 
 class AppTestSSL:
         assert isinstance(_ssl.SSL_ERROR_EOF, int)
         assert isinstance(_ssl.SSL_ERROR_INVALID_ERROR_CODE, int)
 
-        assert is_valid_int(_ssl.OPENSSL_VERSION_NUMBER)
         assert isinstance(_ssl.OPENSSL_VERSION_INFO, tuple)
         assert len(_ssl.OPENSSL_VERSION_INFO) == 5
         assert isinstance(_ssl.OPENSSL_VERSION, str)

pypy/module/cpyext/pystate.py

     [('next', PyInterpreterState)],
     PyInterpreterStateStruct)
 PyThreadState = lltype.Ptr(cpython_struct(
-    "PyThreadState", 
+    "PyThreadState",
     [('interp', PyInterpreterState),
      ('dict', PyObject),
      ]))
 def PyEval_SaveThread(space):
     """Release the global interpreter lock (if it has been created and thread
     support is enabled) and reset the thread state to NULL, returning the
-    previous thread state (which is not NULL except in PyPy).  If the lock has been created,
+    previous thread state.  If the lock has been created,
     the current thread must have acquired it.  (This function is available even
     when thread support is disabled at compile time.)"""
+    state = space.fromcache(InterpreterState)
     if rffi.aroundstate.before:
         rffi.aroundstate.before()
-    return lltype.nullptr(PyThreadState.TO)
+    tstate = state.swap_thread_state(
+        space, lltype.nullptr(PyThreadState.TO))
+    return tstate
 
 @cpython_api([PyThreadState], lltype.Void)
 def PyEval_RestoreThread(space, tstate):
     when thread support is disabled at compile time.)"""
     if rffi.aroundstate.after:
         rffi.aroundstate.after()
+    state = space.fromcache(InterpreterState)
+    state.swap_thread_state(space, tstate)
 
 @cpython_api([], lltype.Void)
 def PyEval_InitThreads(space):
                                   dealloc=ThreadState_dealloc)
 
 from pypy.interpreter.executioncontext import ExecutionContext
+
+# Keep track of the ThreadStateCapsule for a particular execution context.  The
+# default is for new execution contexts not to have one; it is allocated on the
+# first cpyext-based request for it.
 ExecutionContext.cpyext_threadstate = ThreadStateCapsule(None)
 
+# Also keep track of whether it has been initialized yet or not (None is a valid
+# PyThreadState for an execution context to have, when the GIL has been
+# released, so a check against that can't be used to determine the need for
+# initialization).
+ExecutionContext.cpyext_initialized_threadstate = False
+
+def cleanup_cpyext_state(self):
+    try:
+        del self.cpyext_threadstate
+    except AttributeError:
+        pass
+    self.cpyext_initialized_threadstate = False
+ExecutionContext.cleanup_cpyext_state = cleanup_cpyext_state
+
 class InterpreterState(object):
     def __init__(self, space):
         self.interpreter_state = lltype.malloc(
             PyInterpreterState.TO, flavor='raw', zero=True, immortal=True)
 
     def new_thread_state(self, space):
+        """
+        Create a new ThreadStateCapsule to hold the PyThreadState for a
+        particular execution context.
+
+        :param space: A space.
+
+        :returns: A new ThreadStateCapsule holding a newly allocated
+            PyThreadState and referring to this interpreter state.
+        """
         capsule = ThreadStateCapsule(space)
         ts = capsule.memory
         ts.c_interp = self.interpreter_state
         ts.c_dict = make_ref(space, space.newdict())
         return capsule
 
+
     def get_thread_state(self, space):
+        """
+        Get the current PyThreadState for the current execution context.
+
+        :param space: A space.
+
+        :returns: The current PyThreadState for the current execution context,
+            or None if it does not have one.
+        """
         ec = space.getexecutioncontext()
         return self._get_thread_state(space, ec).memory
 
+
+    def swap_thread_state(self, space, tstate):
+        """
+        Replace the current thread state of the current execution context with a
+        new thread state.
+
+        :param space: The space.
+
+        :param tstate: The new PyThreadState for the current execution context.
+
+        :returns: The old thread state for the current execution context, either
+            None or a PyThreadState.
+        """
+        ec = space.getexecutioncontext()
+        capsule = self._get_thread_state(space, ec)
+        old_tstate = capsule.memory
+        capsule.memory = tstate
+        return old_tstate
+
     def _get_thread_state(self, space, ec):
-        if ec.cpyext_threadstate.memory == lltype.nullptr(PyThreadState.TO):
+        """
+        Get the ThreadStateCapsule for the given execution context, possibly
+        creating a new one if it does not already have one.
+
+        :param space: The space.
+        :param ec: The ExecutionContext of which to get the thread state.
+        :returns: The ThreadStateCapsule for the given execution context.
+        """
+        if not ec.cpyext_initialized_threadstate:
             ec.cpyext_threadstate = self.new_thread_state(space)
-
+            ec.cpyext_initialized_threadstate = True
         return ec.cpyext_threadstate
 
 @cpython_api([], PyThreadState, error=CANNOT_FAIL)
 def PyThreadState_Swap(space, tstate):
     """Swap the current thread state with the thread state given by the argument
     tstate, which may be NULL.  The global interpreter lock must be held."""
-    # All cpyext calls release and acquire the GIL, so this function has no
-    # side-effects
-    if tstate:
-        return lltype.nullptr(PyThreadState.TO)
-    else:
-        state = space.fromcache(InterpreterState)
-        return state.get_thread_state(space)
+    state = space.fromcache(InterpreterState)
+    return state.swap_thread_state(space, tstate)
 
 @cpython_api([PyThreadState], lltype.Void)
 def PyEval_AcquireThread(space, tstate):

pypy/module/cpyext/src/getargs.c

 #define FLAG_COMPAT 1
 #define FLAG_SIZE_T 2
 
+typedef int (*destr_t)(PyObject *, void *);
+
+
+/* Keep track of "objects" that have been allocated or initialized and
+   which will need to be deallocated or cleaned up somehow if overall
+   parsing fails.
+*/
+typedef struct {
+  void *item;
+  destr_t destructor;
+} freelistentry_t;
+
+typedef struct {
+  int first_available;
+  freelistentry_t *entries;
+} freelist_t;
+
 
 /* Forward */
 static int vgetargs1(PyObject *, const char *, va_list *, int);
 static void seterror(int, const char *, int *, const char *, const char *);
 static char *convertitem(PyObject *, const char **, va_list *, int, int *, 
-                         char *, size_t, PyObject **);
+                         char *, size_t, freelist_t *);
 static char *converttuple(PyObject *, const char **, va_list *, int,
-			  int *, char *, size_t, int, PyObject **);
+			  int *, char *, size_t, int, freelist_t *);
 static char *convertsimple(PyObject *, const char **, va_list *, int, char *,
-			   size_t, PyObject **);
+			   size_t, freelist_t *);
 static Py_ssize_t convertbuffer(PyObject *, void **p, char **);
 static int getbuffer(PyObject *, Py_buffer *, char**);
 
 
 /* Handle cleanup of allocated memory in case of exception */
 
-static void
-cleanup_ptr(void *ptr)
+static int
+cleanup_ptr(PyObject *self, void *ptr)
 {
-	PyMem_FREE(ptr);
-}
-
-static void
-cleanup_buffer(void *ptr)
-{
-	PyBuffer_Release((Py_buffer *) ptr);
+    if (ptr) {
+        PyMem_FREE(ptr);
+    }
+    return 0;
 }
 
 static int
-addcleanup(void *ptr, PyObject **freelist, void (*destr)(void *))
+cleanup_buffer(PyObject *self, void *ptr)
 {
-	PyObject *cobj;
-	if (!*freelist) {
-		*freelist = PyList_New(0);
-		if (!*freelist) {
-			destr(ptr);
-			return -1;
-		}
-	}
-	cobj = PyCObject_FromVoidPtr(ptr, destr);
-	if (!cobj) {
-		destr(ptr);
-		return -1;
-	}
-	if (PyList_Append(*freelist, cobj)) {
-		Py_DECREF(cobj);
-		return -1;
-	}
-        Py_DECREF(cobj);
-	return 0;
+    Py_buffer *buf = (Py_buffer *)ptr;
+    if (buf) {
+        PyBuffer_Release(buf);
+    }
+    return 0;
 }
 
 static int
-cleanreturn(int retval, PyObject *freelist)
+addcleanup(void *ptr, freelist_t *freelist, destr_t destructor)
 {
-	if (freelist && retval != 0) {
-		/* We were successful, reset the destructors so that they
-		   don't get called. */
-		Py_ssize_t len = PyList_GET_SIZE(freelist), i;
-		for (i = 0; i < len; i++)
-			((PyCObject *) PyList_GET_ITEM(freelist, i))
-				->destructor = NULL;
-	}
-	Py_XDECREF(freelist);
-	return retval;
+    int index;
+
+    index = freelist->first_available;
+    freelist->first_available += 1;
+
+    freelist->entries[index].item = ptr;
+    freelist->entries[index].destructor = destructor;
+
+    return 0;
 }
 
+static int
+cleanreturn(int retval, freelist_t *freelist)
+{
+    int index;
+
+    if (retval == 0) {
+      /* A failure occurred, therefore execute all of the cleanup
+	 functions.
+      */
+      for (index = 0; index < freelist->first_available; ++index) {
+          freelist->entries[index].destructor(NULL,
+                                              freelist->entries[index].item);
+      }
+    }
+    PyMem_Free(freelist->entries);
+    return retval;
+}
 
 static int
 vgetargs1(PyObject *args, const char *format, va_list *p_va, int flags)
 	const char *formatsave = format;
 	Py_ssize_t i, len;
 	char *msg;
-	PyObject *freelist = NULL;
+	freelist_t freelist = {0, NULL};
 	int compat = flags & FLAG_COMPAT;
 
 	assert(compat || (args != (PyObject*)NULL));
 	
 	format = formatsave;
 	
+	freelist.entries = PyMem_New(freelistentry_t, max);
+
 	if (compat) {
 		if (max == 0) {
 			if (args == NULL)
-				return 1;
+			    return cleanreturn(1, &freelist);
 			PyOS_snprintf(msgbuf, sizeof(msgbuf),
 				      "%.200s%s takes no arguments",
 				      fname==NULL ? "function" : fname,
 				      fname==NULL ? "" : "()");
 			PyErr_SetString(PyExc_TypeError, msgbuf);
-			return 0;
+			return cleanreturn(0, &freelist);
 		}
 		else if (min == 1 && max == 1) {
 			if (args == NULL) {
 					      fname==NULL ? "function" : fname,
 					      fname==NULL ? "" : "()");
 				PyErr_SetString(PyExc_TypeError, msgbuf);
-				return 0;
+				return cleanreturn(0, &freelist);
 			}
 			msg = convertitem(args, &format, p_va, flags, levels, 
 					  msgbuf, sizeof(msgbuf), &freelist);
 			if (msg == NULL)
-				return cleanreturn(1, freelist);
+				return cleanreturn(1, &freelist);
 			seterror(levels[0], msg, levels+1, fname, message);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 		else {
 			PyErr_SetString(PyExc_SystemError,
 			    "old style getargs format uses new features");
-			return 0;
+			return cleanreturn(0, &freelist);
 		}
 	}
 	
 	if (!PyTuple_Check(args)) {
 		PyErr_SetString(PyExc_SystemError,
 		    "new style getargs format but argument is not a tuple");
-		return 0;
+		return cleanreturn(0, &freelist);
 	}
 	
 	len = PyTuple_GET_SIZE(args);
 			message = msgbuf;
 		}
 		PyErr_SetString(PyExc_TypeError, message);
-		return 0;
+		return cleanreturn(0, &freelist);
 	}
 	
 	for (i = 0; i < len; i++) {
 				  sizeof(msgbuf), &freelist);
 		if (msg) {
 			seterror(i+1, msg, levels, fname, message);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 	}
 
 	    *format != '|' && *format != ':' && *format != ';') {
 		PyErr_Format(PyExc_SystemError,
 			     "bad format string: %.200s", formatsave);
-		return cleanreturn(0, freelist);
+		return cleanreturn(0, &freelist);
 	}
 	
-	return cleanreturn(1, freelist);
+	return cleanreturn(1, &freelist);
 }
 
 
 static char *
 converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
              int *levels, char *msgbuf, size_t bufsize, int toplevel, 
-             PyObject **freelist)
+             freelist_t *freelist)
 {
 	int level = 0;
 	int n = 0;
 
 static char *
 convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags,
-            int *levels, char *msgbuf, size_t bufsize, PyObject **freelist)
+            int *levels, char *msgbuf, size_t bufsize, freelist_t *freelist)
 {
 	char *msg;
 	const char *format = *p_format;
 
 static char *
 convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
-              char *msgbuf, size_t bufsize, PyObject **freelist)
+              char *msgbuf, size_t bufsize, freelist_t *freelist)
 {
 	/* For # codes */
 #define FETCH_SIZE	int *q=NULL;Py_ssize_t *q2=NULL;\
 	const char *fname, *msg, *custom_msg, *keyword;
 	int min = INT_MAX;
 	int i, len, nargs, nkeywords;
-	PyObject *freelist = NULL, *current_arg;
+	PyObject *current_arg;
+	freelist_t freelist = {0, NULL};
+
 
 	assert(args != NULL && PyTuple_Check(args));
 	assert(keywords == NULL || PyDict_Check(keywords));
 	for (len=0; kwlist[len]; len++)
 		continue;
 
+	freelist.entries = PyMem_New(freelistentry_t, len);
+
 	nargs = PyTuple_GET_SIZE(args);
 	nkeywords = (keywords == NULL) ? 0 : PyDict_Size(keywords);
 	if (nargs + nkeywords > len) {
 			     len,
 			     (len == 1) ? "" : "s",
 			     nargs + nkeywords);
-		return 0;
+		return cleanreturn(0, &freelist);
 	}
 
 	/* convert tuple args and keyword args in same loop, using kwlist to drive process */
 			PyErr_Format(PyExc_RuntimeError,
 				     "More keyword list entries (%d) than "
 				     "format specifiers (%d)", len, i);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 		current_arg = NULL;
 		if (nkeywords) {
 					     "Argument given by name ('%s') "
 					     "and position (%d)",
 					     keyword, i+1);
-				return cleanreturn(0, freelist);
+				return cleanreturn(0, &freelist);
 			}
 		}
 		else if (nkeywords && PyErr_Occurred())
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		else if (i < nargs)
 			current_arg = PyTuple_GET_ITEM(args, i);
 			
 				levels, msgbuf, sizeof(msgbuf), &freelist);
 			if (msg) {
 				seterror(i+1, msg, levels, fname, custom_msg);
-				return cleanreturn(0, freelist);
+				return cleanreturn(0, &freelist);
 			}
 			continue;
 		}
 			PyErr_Format(PyExc_TypeError, "Required argument "
 				     "'%s' (pos %d) not found",
 				     keyword, i+1);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 		/* current code reports success when all required args
 		 * fulfilled and no keyword args left, with no further
 		 * validation. XXX Maybe skip this in debug build ?
 		 */
 		if (!nkeywords)
-			return cleanreturn(1, freelist);
+			return cleanreturn(1, &freelist);
 
 		/* We are into optional args, skip thru to any remaining
 		 * keyword args */
 		if (msg) {
 			PyErr_Format(PyExc_RuntimeError, "%s: '%s'", msg,
 				     format);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 	}
 
 		PyErr_Format(PyExc_RuntimeError,
 			"more argument specifiers than keyword list entries "
 			"(remaining format:'%s')", format);
-		return cleanreturn(0, freelist);
+		return cleanreturn(0, &freelist);
 	}
 
 	/* make sure there are no extraneous keyword arguments */
 			if (!PyString_Check(key)) {
                             PyErr_SetString(PyExc_TypeError, 
 					        "keywords must be strings");
-				return cleanreturn(0, freelist);
+				return cleanreturn(0, &freelist);
 			}
 			ks = PyString_AsString(key);
 			for (i = 0; i < len; i++) {
 					     "'%s' is an invalid keyword "
 					     "argument for this function",
 					     ks);
-				return cleanreturn(0, freelist);
+				return cleanreturn(0, &freelist);
 			}
 		}
 	}
 
-	return cleanreturn(1, freelist);
+	return cleanreturn(1, &freelist);
 }
 
 

pypy/module/cpyext/test/test_cpyext.py

             del obj
         import gc; gc.collect()
 
-        try:
-            del space.getexecutioncontext().cpyext_threadstate
-        except AttributeError:
-            pass
+        space.getexecutioncontext().cleanup_cpyext_state()
 
         for w_obj in state.non_heaptypes_w:
             Py_DecRef(space, w_obj)

pypy/module/cpyext/test/test_pystate.py

 from pypy.rpython.lltypesystem.lltype import nullptr
 from pypy.module.cpyext.pystate import PyInterpreterState, PyThreadState
 from pypy.module.cpyext.pyobject import from_ref
+from pypy.rpython.lltypesystem import lltype
+from pypy.module.cpyext.test.test_cpyext import LeakCheckingTest, freeze_refcnts
+from pypy.module.cpyext.pystate import PyThreadState_Get, PyInterpreterState_Head
+from pypy.tool import leakfinder
 
 class AppTestThreads(AppTestCpythonExtensionBase):
     def test_allow_threads(self):
         # Should compile at least
         module.test()
 
+
+    def test_thread_state_get(self):
+        module = self.import_extension('foo', [
+                ("get", "METH_NOARGS",
+                 """
+                     PyThreadState *tstate = PyThreadState_Get();
+                     if (tstate == NULL) {
+                         return PyLong_FromLong(0);
+                     }
+                     if (tstate->interp != PyInterpreterState_Head()) {
+                         return PyLong_FromLong(1);
+                     }
+                     if (tstate->interp->next != NULL) {
+                         return PyLong_FromLong(2);
+                     }
+                     return PyLong_FromLong(3);
+                 """),
+                ])
+        assert module.get() == 3
+
+    def test_basic_threadstate_dance(self):
+        module = self.import_extension('foo', [
+                ("dance", "METH_NOARGS",
+                 """
+                     PyThreadState *old_tstate, *new_tstate;
+
+                     old_tstate = PyThreadState_Swap(NULL);
+                     if (old_tstate == NULL) {
+                         return PyLong_FromLong(0);
+                     }
+
+                     new_tstate = PyThreadState_Get();
+                     if (new_tstate != NULL) {
+                         return PyLong_FromLong(1);
+                     }
+
+                     new_tstate = PyThreadState_Swap(old_tstate);
+                     if (new_tstate != NULL) {
+                         return PyLong_FromLong(2);
+                     }
+
+                     new_tstate = PyThreadState_Get();
+                     if (new_tstate != old_tstate) {
+                         return PyLong_FromLong(3);
+                     }
+
+                     return PyLong_FromLong(4);
+                 """),
+                ])
+        assert module.dance() == 4
+
+    def test_threadstate_dict(self):
+        module = self.import_extension('foo', [
+                ("getdict", "METH_NOARGS",
+                 """
+                 PyObject *dict = PyThreadState_GetDict();
+                 Py_INCREF(dict);
+                 return dict;
+                 """),
+                ])
+        assert isinstance(module.getdict(), dict)
+
+    def test_savethread(self):
+        module = self.import_extension('foo', [
+                ("bounce", "METH_NOARGS",
+                 """
+                 PyThreadState *tstate = PyEval_SaveThread();
+                 if (tstate == NULL) {
+                     return PyLong_FromLong(0);
+                 }
+
+                 if (PyThreadState_Get() != NULL) {
+                     return PyLong_FromLong(1);
+                 }
+
+                 PyEval_RestoreThread(tstate);
+
+                 if (PyThreadState_Get() != tstate) {
+                     return PyLong_FromLong(2);
+                 }
+
+                 return PyLong_FromLong(3);
+                                  """),
+                ])
+
+
+
 class TestInterpreterState(BaseApiTest):
     def test_interpreter_head(self, space, api):
         state = api.PyInterpreterState_Head()
     def test_interpreter_next(self, space, api):
         state = api.PyInterpreterState_Head()
         assert nullptr(PyInterpreterState.TO) == api.PyInterpreterState_Next(state)
-
-class TestThreadState(BaseApiTest):
-    def test_thread_state_get(self, space, api):
-        ts = api.PyThreadState_Get()
-        assert ts != nullptr(PyThreadState.TO)
-
-    def test_thread_state_interp(self, space, api):
-        ts = api.PyThreadState_Get()
-        assert ts.c_interp == api.PyInterpreterState_Head()
-        assert ts.c_interp.c_next == nullptr(PyInterpreterState.TO)
-
-    def test_basic_threadstate_dance(self, space, api):
-        # Let extension modules call these functions,
-        # Not sure of the semantics in pypy though.
-        # (cpyext always acquires and releases the GIL around calls)
-        tstate = api.PyThreadState_Swap(None)
-        assert tstate is not None
-        assert not api.PyThreadState_Swap(tstate)
-
-        api.PyEval_AcquireThread(tstate)
-        api.PyEval_ReleaseThread(tstate)
-
-    def test_threadstate_dict(self, space, api):
-        ts = api.PyThreadState_Get()
-        ref = ts.c_dict
-        assert ref == api.PyThreadState_GetDict()
-        w_obj = from_ref(space, ref)
-        assert space.isinstance_w(w_obj, space.w_dict)

pypy/module/math/test/test_direct.py

         ('copysign', (1.5, -0.0), -1.5),
         ('copysign', (1.5, INFINITY), 1.5),
         ('copysign', (1.5, -INFINITY), -1.5),
+        ]
+    if sys.platform != 'win32':    # all NaNs seem to be negative there...?
+        IRREGCASES += [
         ('copysign', (1.5, NAN), 1.5),
         ('copysign', (1.75, -NAN), -1.75),      # special case for -NAN here
         ]

pypy/module/micronumpy/__init__.py

         'True_': 'types.Bool.True',
         'False_': 'types.Bool.False',
 
+        'typeinfo': 'interp_dtype.get_dtype_cache(space).w_typeinfo',
+
         'generic': 'interp_boxes.W_GenericBox',
         'number': 'interp_boxes.W_NumberBox',
         'integer': 'interp_boxes.W_IntegerBox',
         'signedinteger': 'interp_boxes.W_SignedIntegerBox',
         'unsignedinteger': 'interp_boxes.W_UnsignedIntegerBox',
         'bool_': 'interp_boxes.W_BoolBox',
+        'bool8': 'interp_boxes.W_BoolBox',
         'int8': 'interp_boxes.W_Int8Box',
+        'byte': 'interp_boxes.W_Int8Box',
         'uint8': 'interp_boxes.W_UInt8Box',
+        'ubyte': 'interp_boxes.W_UInt8Box',
         'int16': 'interp_boxes.W_Int16Box',
+        'short': 'interp_boxes.W_Int16Box',
         'uint16': 'interp_boxes.W_UInt16Box',
+        'ushort': 'interp_boxes.W_UInt16Box',
         'int32': 'interp_boxes.W_Int32Box',
+        'intc': 'interp_boxes.W_Int32Box',
         'uint32': 'interp_boxes.W_UInt32Box',
+        'uintc': 'interp_boxes.W_UInt32Box',
         'int64': 'interp_boxes.W_Int64Box',
         'uint64': 'interp_boxes.W_UInt64Box',
+        'longlong': 'interp_boxes.W_LongLongBox',
+        'ulonglong': 'interp_boxes.W_ULongLongBox',
         'int_': 'interp_boxes.W_LongBox',
         'inexact': 'interp_boxes.W_InexactBox',
         'floating': 'interp_boxes.W_FloatingBox',
         'float_': 'interp_boxes.W_Float64Box',
         'float32': 'interp_boxes.W_Float32Box',
         'float64': 'interp_boxes.W_Float64Box',
+        'intp': 'types.IntP.BoxType',
+        'uintp': 'types.UIntP.BoxType',
+        'flexible': 'interp_boxes.W_FlexibleBox',
+        'character': 'interp_boxes.W_CharacterBox',
+        'str_': 'interp_boxes.W_StringBox',
+        'unicode_': 'interp_boxes.W_UnicodeBox',
+        'void': 'interp_boxes.W_VoidBox',
     }
 
     # ufuncs
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arctan2", "arctan2"),
         ("arccosh", "arccosh"),
         ("arcsinh", "arcsinh"),
         ("arctanh", "arctanh"),
         ("true_divide", "true_divide"),
         ("equal", "equal"),
         ("exp", "exp"),
+        ("exp2", "exp2"),
+        ("expm1", "expm1"),
         ("fabs", "fabs"),
+        ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
         ("greater", "greater"),
         ("radians", "radians"),
         ("degrees", "degrees"),
         ("deg2rad", "radians"),
+        ("rad2deg", "degrees"),
         ("reciprocal", "reciprocal"),
         ("sign", "sign"),
+        ("signbit", "signbit"),
         ("sin", "sin"),
         ("sinh", "sinh"),
         ("subtract", "subtract"),
         ('bitwise_not', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
+        ('isneginf', 'isneginf'),
+        ('isposinf', 'isposinf'),
+        ('isfinite', 'isfinite'),
         ('logical_and', 'logical_and'),
         ('logical_xor', 'logical_xor'),
         ('logical_not', 'logical_not'),
         ('log1p', 'log1p'),
         ('power', 'power'),
         ('floor_divide', 'floor_divide'),
+        ('logaddexp', 'logaddexp'),
+        ('logaddexp2', 'logaddexp2'),
     ]:
         interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl
 

pypy/module/micronumpy/compile.py

     pass
 
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
-                        "unegative", "flat"]
+                        "unegative", "flat", "tostring"]
 TWO_ARG_FUNCTIONS = ["dot", 'take']
 
 class FakeSpace(object):
     w_long = "long"
     w_tuple = 'tuple'
     w_slice = "slice"
+    w_str = "str"
+    w_unicode = "unicode"
 
     def __init__(self):
         """NOT_RPYTHON"""
             return BoolObject(obj)
         elif isinstance(obj, int):
             return IntObject(obj)
+        elif isinstance(obj, long):
+            return LongObject(obj)
         elif isinstance(obj, W_Root):
             return obj
+        elif isinstance(obj, str):
+            return StringObject(obj)
         raise NotImplementedError
 
     def newlist(self, items):
             return int(w_obj.floatval)
         raise NotImplementedError
 
+    def str_w(self, w_obj):
+        if isinstance(w_obj, StringObject):
+            return w_obj.v
+        raise NotImplementedError
+
     def int(self, w_obj):
         if isinstance(w_obj, IntObject):
             return w_obj
         return instantiate(klass)
 
     def newtuple(self, list_w):
-        raise ValueError
+        return ListObject(list_w)
+
+    def newdict(self):
+        return {}
+
+    def setitem(self, dict, item, value):
+        dict[item] = value
 
     def len_w(self, w_obj):
         if isinstance(w_obj, ListObject):
     def __init__(self, intval):
         self.intval = intval
 
+class LongObject(W_Root):
+    tp = FakeSpace.w_long
+    def __init__(self, intval):
+        self.intval = intval
+
 class ListObject(W_Root):
     tp = FakeSpace.w_list
     def __init__(self, items):
         self.stop = stop
         self.step = step
 
+class StringObject(W_Root):
+    tp = FakeSpace.w_str
+    def __init__(self, v):
+        self.v = v
+
 class InterpreterState(object):
     def __init__(self, code):
         self.code = code
                 w_res = neg.call(interp.space, [arr])
             elif self.name == "flat":
                 w_res = arr.descr_get_flatiter(interp.space)
+            elif self.name == "tostring":
+                arr.descr_tostring(interp.space)
+                w_res = None
             else:
                 assert False # unreachable code
         elif self.name in TWO_ARG_FUNCTIONS:

pypy/module/micronumpy/interp_boxes.py

 from pypy.interpreter.baseobjspace import Wrappable
-from pypy.interpreter.error import operationerrfmt
+from pypy.interpreter.error import operationerrfmt, OperationError
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.floattype import float_typedef
+from pypy.objspace.std.stringtype import str_typedef
+from pypy.objspace.std.unicodetype import unicode_typedef, unicode_from_object
 from pypy.objspace.std.inttype import int_typedef
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
-
 MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
 MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
 
 def new_dtype_getter(name):
-    def get_dtype(space):
+    def _get_dtype(space):
         from pypy.module.micronumpy.interp_dtype import get_dtype_cache
         return getattr(get_dtype_cache(space), "w_%sdtype" % name)
     def new(space, w_subtype, w_value):
-        dtype = get_dtype(space)
+        dtype = _get_dtype(space)
         return dtype.itemtype.coerce_subtype(space, w_subtype, w_value)
-    return func_with_new_name(new, name + "_box_new"), staticmethod(get_dtype)
+    return func_with_new_name(new, name + "_box_new"), staticmethod(_get_dtype)
 
 class PrimitiveBox(object):
     _mixin_ = True
             w_subtype.getname(space, '?')
         )
 
+    def get_dtype(self, space):
+        return self._get_dtype(space)
+
     def descr_str(self, space):
         return space.wrap(self.get_dtype(space).itemtype.str_format(self))
 
         return space.format(self.item(space), w_spec)
 
     def descr_int(self, space):
-        box = self.convert_to(W_LongBox.get_dtype(space))
+        box = self.convert_to(W_LongBox._get_dtype(space))
         assert isinstance(box, W_LongBox)
         return space.wrap(box.value)
 
     def descr_float(self, space):
-        box = self.convert_to(W_Float64Box.get_dtype(space))
+        box = self.convert_to(W_Float64Box._get_dtype(space))
         assert isinstance(box, W_Float64Box)
         return space.wrap(box.value)
 
 
 
 class W_BoolBox(W_GenericBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("bool")
+    descr__new__, _get_dtype = new_dtype_getter("bool")
 
 class W_NumberBox(W_GenericBox):
     _attrs_ = ()
     pass
 
 class W_Int8Box(W_SignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("int8")
+    descr__new__, _get_dtype = new_dtype_getter("int8")
 
 class W_UInt8Box(W_UnsignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("uint8")
+    descr__new__, _get_dtype = new_dtype_getter("uint8")
 
 class W_Int16Box(W_SignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("int16")
+    descr__new__, _get_dtype = new_dtype_getter("int16")
 
 class W_UInt16Box(W_UnsignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("uint16")
+    descr__new__, _get_dtype = new_dtype_getter("uint16")
 
 class W_Int32Box(W_SignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("int32")
+    descr__new__, _get_dtype = new_dtype_getter("int32")
 
 class W_UInt32Box(W_UnsignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("uint32")
+    descr__new__, _get_dtype = new_dtype_getter("uint32")
 
 class W_LongBox(W_SignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("long")
+    descr__new__, _get_dtype = new_dtype_getter("long")
 
 class W_ULongBox(W_UnsignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("ulong")
+    descr__new__, _get_dtype = new_dtype_getter("ulong")
 
 class W_Int64Box(W_SignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("int64")
+    descr__new__, _get_dtype = new_dtype_getter("int64")
+
+class W_LongLongBox(W_SignedIntegerBox, PrimitiveBox):
+    descr__new__, _get_dtype = new_dtype_getter('longlong')
 
 class W_UInt64Box(W_UnsignedIntegerBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("uint64")
+    descr__new__, _get_dtype = new_dtype_getter("uint64")
+
+class W_ULongLongBox(W_SignedIntegerBox, PrimitiveBox):
+    descr__new__, _get_dtype = new_dtype_getter('ulonglong')
 
 class W_InexactBox(W_NumberBox):
     _attrs_ = ()
     _attrs_ = ()
 
 class W_Float32Box(W_FloatingBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("float32")
+    descr__new__, _get_dtype = new_dtype_getter("float32")
 
 class W_Float64Box(W_FloatingBox, PrimitiveBox):
-    descr__new__, get_dtype = new_dtype_getter("float64")
+    descr__new__, _get_dtype = new_dtype_getter("float64")
 
 
+class W_FlexibleBox(W_GenericBox):
+    def __init__(self, arr, ofs, dtype):
+        self.arr = arr # we have to keep array alive
+        self.ofs = ofs
+        self.dtype = dtype
+
+    def get_dtype(self, space):
+        return self.arr.dtype
+
 @unwrap_spec(self=W_GenericBox)
 def descr_index(space, self):
     return space.index(self.item(space))
 
+class W_VoidBox(W_FlexibleBox):
+    @unwrap_spec(item=str)
+    def descr_getitem(self, space, item):
+        try:
+            ofs, dtype = self.dtype.fields[item]
+        except KeyError:
+            raise OperationError(space.w_IndexError,
+                                 space.wrap("Field %s does not exist" % item))
+        return dtype.itemtype.read(self.arr, 1, self.ofs, ofs, dtype)
+
+    @unwrap_spec(item=str)
+    def descr_setitem(self, space, item, w_value):
+        try:
+            ofs, dtype = self.dtype.fields[item]
+        except KeyError:
+            raise OperationError(space.w_IndexError,
+                                 space.wrap("Field %s does not exist" % item))
+        dtype.itemtype.store(self.arr, 1, self.ofs, ofs,
+                             dtype.coerce(space, w_value))
+
+class W_CharacterBox(W_FlexibleBox):
+    pass
+
+class W_StringBox(W_CharacterBox):
+    def descr__new__string_box(space, w_subtype, w_arg):
+        from pypy.module.micronumpy.interp_numarray import W_NDimArray
+        from pypy.module.micronumpy.interp_dtype import new_string_dtype
+
+        arg = space.str_w(space.str(w_arg))
+        arr = W_NDimArray([1], new_string_dtype(space, len(arg)))
+        for i in range(len(arg)):
+            arr.storage[i] = arg[i]
+        return W_StringBox(arr, 0, arr.dtype)
+
+
+class W_UnicodeBox(W_CharacterBox):
+    def descr__new__unicode_box(space, w_subtype, w_arg):
+        from pypy.module.micronumpy.interp_numarray import W_NDimArray
+        from pypy.module.micronumpy.interp_dtype import new_unicode_dtype
+
+        arg = space.unicode_w(unicode_from_object(space, w_arg))
+        arr = W_NDimArray([1], new_unicode_dtype(space, len(arg)))
+        # XXX not this way, we need store
+        #for i in range(len(arg)):
+        #    arr.storage[i] = arg[i]
+        return W_UnicodeBox(arr, 0, arr.dtype)
 
 W_GenericBox.typedef = TypeDef("generic",
     __module__ = "numpypy",
     __new__ = interp2app(W_Float64Box.descr__new__.im_func),
 )
 
+
+W_FlexibleBox.typedef = TypeDef("flexible", W_GenericBox.typedef,
+    __module__ = "numpypy",
+)
+
+W_VoidBox.typedef = TypeDef("void", W_FlexibleBox.typedef,
+    __module__ = "numpypy",
+    __getitem__ = interp2app(W_VoidBox.descr_getitem),
+    __setitem__ = interp2app(W_VoidBox.descr_setitem),
+)
+
+W_CharacterBox.typedef = TypeDef("character", W_FlexibleBox.typedef,
+    __module__ = "numpypy",
+)
+
+W_StringBox.typedef = TypeDef("string_", (str_typedef, W_CharacterBox.typedef),
+    __module__ = "numpypy",
+    __new__ = interp2app(W_StringBox.descr__new__string_box.im_func),
+)
+
+W_UnicodeBox.typedef = TypeDef("unicode_", (unicode_typedef, W_CharacterBox.typedef),
+    __module__ = "numpypy",
+    __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func),
+)
+                                          

pypy/module/micronumpy/interp_dtype.py

+
+import sys
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
     interp_attrproperty, interp_attrproperty_w)
-from pypy.module.micronumpy import types, signature, interp_boxes
+from pypy.module.micronumpy import types, interp_boxes
 from pypy.rlib.objectmodel import specialize
-from pypy.rlib.rarithmetic import LONG_BIT
-from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong
 
 
 UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
-
-
-VOID_STORAGE = lltype.Array(lltype.Char, hints={'nolength': True, 'render_as_void': True})
+VOIDLTR = 'V'
+STRINGLTR = 'S'
+UNICODELTR = 'U'
 
 class W_Dtype(Wrappable):
     _immutable_fields_ = ["itemtype", "num", "kind"]
 
-    def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]):
+    def __init__(self, itemtype, num, kind, name, char, w_box_type,
+                 alternate_constructors=[], aliases=[],
+                 fields=None, fieldnames=None, native=True):
         self.itemtype = itemtype
         self.num = num
         self.kind = kind
         self.w_box_type = w_box_type
         self.alternate_constructors = alternate_constructors
         self.aliases = aliases
-
-    def malloc(self, length):
-        # XXX find out why test_zjit explodes with tracking of allocations
-        return lltype.malloc(VOID_STORAGE, self.itemtype.get_element_size() * length,
-            zero=True, flavor="raw",
-            track_allocation=False, add_memory_pressure=True
-        )
+        self.fields = fields
+        self.fieldnames = fieldnames
+        self.native = native
 
     @specialize.argtype(1)
     def box(self, value):
         return self.itemtype.box(value)
 
     def coerce(self, space, w_item):
-        return self.itemtype.coerce(space, w_item)
+        return self.itemtype.coerce(space, self, w_item)
 
-    def getitem(self, storage, i):
-        return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0)
+    def getitem(self, arr, i):
+        return self.itemtype.read(arr, 1, i, 0)
 
-    def getitem_bool(self, storage, i):
-        isize = self.itemtype.get_element_size()
-        return self.itemtype.read_bool(storage, isize, i, 0)
+    def getitem_bool(self, arr, i):
+        return self.itemtype.read_bool(arr, 1, i, 0)
 
-    def setitem(self, storage, i, box):
-        self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box)
+    def setitem(self, arr, i, box):
+        self.itemtype.store(arr, 1, i, 0, box)
 
     def fill(self, storage, box, start, stop):
-        self.itemtype.fill(storage, self.itemtype.get_element_size(), box, start, stop, 0)
-
-    def descr__new__(space, w_subtype, w_dtype):
-        cache = get_dtype_cache(space)
-
-        if space.is_w(w_dtype, space.w_None):
-            return cache.w_float64dtype
-        elif space.isinstance_w(w_dtype, w_subtype):
-            return w_dtype
-        elif space.isinstance_w(w_dtype, space.w_str):
-            name = space.str_w(w_dtype)
-            for dtype in cache.builtin_dtypes:
-                if dtype.name == name or dtype.char == name or name in dtype.aliases:
-                    return dtype
-        else:
-            for dtype in cache.builtin_dtypes:
-                if w_dtype in dtype.alternate_constructors:
-                    return dtype
-                if w_dtype is dtype.w_box_type:
-                    return dtype
-        raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+        self.itemtype.fill(storage, self.get_size(), box, start, stop, 0)
 
     def descr_str(self, space):
         return space.wrap(self.name)
     def descr_get_itemsize(self, space):
         return space.wrap(self.itemtype.get_element_size())
 
+    def descr_get_byteorder(self, space):
+        if self.native:
+            return space.wrap('=')
+        return space.wrap(nonnative_byteorder_prefix)
+
+    def descr_get_alignment(self, space):
+        return space.wrap(self.itemtype.alignment)
+
     def descr_get_shape(self, space):
         return space.newtuple([])
 
     def descr_ne(self, space, w_other):
         return space.wrap(not self.eq(space, w_other))
 
+    def descr_get_fields(self, space):
+        if self.fields is None:
+            return space.w_None
+        w_d = space.newdict()
+        for name, (offset, subdtype) in self.fields.iteritems():
+            space.setitem(w_d, space.wrap(name), space.newtuple([subdtype,
+                                                                 space.wrap(offset)]))
+        return w_d
+
+    def descr_get_names(self, space):
+        if self.fieldnames is None:
+            return space.w_None
+        return space.newtuple([space.wrap(name) for name in self.fieldnames])
+
+    @unwrap_spec(item=str)
+    def descr_getitem(self, space, item):
+        if self.fields is None:
+            raise OperationError(space.w_KeyError, space.wrap("There are no keys in dtypes %s" % self.name))
+        try:
+            return self.fields[item][1]
+        except KeyError:
+            raise OperationError(space.w_KeyError, space.wrap("Field named %s not found" % item))
+
     def is_int_type(self):
         return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or
                 self.kind == BOOLLTR)
 
+    def is_signed(self):
+        return self.kind == SIGNEDLTR
+
     def is_bool_type(self):
         return self.kind == BOOLLTR
 
+    def is_record_type(self):
+        return self.fields is not None
+
+    def __repr__(self):
+        if self.fields is not None:
+            return '<DType %r>' % self.fields
+        return '<DType %r>' % self.itemtype
+
+    def get_size(self):
+        return self.itemtype.get_element_size()
+
+def dtype_from_list(space, w_lst):
+    lst_w = space.listview(w_lst)
+    fields = {}
+    offset = 0
+    ofs_and_items = []
+    fieldnames = []
+    for w_elem in lst_w:
+        w_fldname, w_flddesc = space.fixedview(w_elem, 2)
+        subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc)
+        fldname = space.str_w(w_fldname)
+        if fldname in fields:
+            raise OperationError(space.w_ValueError, space.wrap("two fields with the same name"))
+        assert isinstance(subdtype, W_Dtype)
+        fields[fldname] = (offset, subdtype)
+        ofs_and_items.append((offset, subdtype.itemtype))
+        offset += subdtype.itemtype.get_element_size()
+        fieldnames.append(fldname)
+    itemtype = types.RecordType(ofs_and_items, offset)
+    return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
+                   "V", space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
+                   fieldnames=fieldnames)
+
+def dtype_from_dict(space, w_dict):
+    raise OperationError(space.w_NotImplementedError, space.wrap(
+        "dtype from dict"))
+
+def variable_dtype(space, name):
+    if name[0] in '<>=':
+        name = name[1:]
+    char = name[0]
+    if len(name) == 1:
+        size = 0
+    else:
+        try:
+            size = int(name[1:])
+        except ValueError:
+            raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+    if char == 'S':
+        itemtype = types.StringType(size)
+        basename = 'string'
+        num = 18
+        w_box_type = space.gettypefor(interp_boxes.W_StringBox)
+    elif char == 'V':
+        num = 20
+        basename = 'void'
+        w_box_type = space.gettypefor(interp_boxes.W_VoidBox)
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "pure void dtype"))
+    else:
+        assert char == 'U'
+        basename = 'unicode'
+        itemtype = types.UnicodeType(size)
+        num = 19
+        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
+    return W_Dtype(itemtype, num, char,
+                   basename + str(8 * itemtype.get_element_size()),
+                   char, w_box_type)
+
+def dtype_from_spec(space, name):
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "dtype from spec"))    
+
+def descr__new__(space, w_subtype, w_dtype):
+    cache = get_dtype_cache(space)
+
+    if space.is_w(w_dtype, space.w_None):
+        return cache.w_float64dtype
+    elif space.isinstance_w(w_dtype, w_subtype):
+        return w_dtype
+    elif space.isinstance_w(w_dtype, space.w_str):
+        name = space.str_w(w_dtype)
+        if ',' in name:
+            return dtype_from_spec(space, name)
+        try:
+            return cache.dtypes_by_name[name]
+        except KeyError:
+            pass
+        if name[0] in 'VSU' or name[0] in '<>=' and name[1] in 'VSU':
+            return variable_dtype(space, name)
+    elif space.isinstance_w(w_dtype, space.w_list):
+        return dtype_from_list(space, w_dtype)
+    elif space.isinstance_w(w_dtype, space.w_dict):
+        return dtype_from_dict(space, w_dtype)
+    else:
+        for dtype in cache.builtin_dtypes:
+            if w_dtype in dtype.alternate_constructors:
+                return dtype
+            if w_dtype is dtype.w_box_type:
+                return dtype
+    raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+
 W_Dtype.typedef = TypeDef("dtype",
     __module__ = "numpypy",
-    __new__ = interp2app(W_Dtype.descr__new__.im_func),
+    __new__ = interp2app(descr__new__),
 
     __str__= interp2app(W_Dtype.descr_str),
     __repr__ = interp2app(W_Dtype.descr_repr),
     __eq__ = interp2app(W_Dtype.descr_eq),
     __ne__ = interp2app(W_Dtype.descr_ne),
+    __getitem__ = interp2app(W_Dtype.descr_getitem),
 
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
+    char = interp_attrproperty("char", cls=W_Dtype),
     type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
+    byteorder = GetSetProperty(W_Dtype.descr_get_byteorder),
     itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
+    alignment = GetSetProperty(W_Dtype.descr_get_alignment),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
     name = interp_attrproperty('name', cls=W_Dtype),
+    fields = GetSetProperty(W_Dtype.descr_get_fields),
+    names = GetSetProperty(W_Dtype.descr_get_names),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
 
+if sys.byteorder == 'little':
+    byteorder_prefix = '<'
+    nonnative_byteorder_prefix = '>'
+else:
+    byteorder_prefix = '>'
+    nonnative_byteorder_prefix = '<'
+
+def new_string_dtype(space, size):
+    return W_Dtype(
+        types.StringType(size),
+        num=18,
+        kind=STRINGLTR,
+        name='string',
+        char='S' + str(size),
+        w_box_type = space.gettypefor(interp_boxes.W_StringBox),
+    )
+
+def new_unicode_dtype(space, size):
+    return W_Dtype(
+        types.UnicodeType(size),
+        num=19,
+        kind=UNICODELTR,
+        name='unicode',
+        char='U' + str(size),
+        w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
+    )
+
+
 class DtypeCache(object):
     def __init__(self, space):
         self.w_booldtype = W_Dtype(
             name="int64",
             char="q",
             w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
-            alternate_constructors=[space.w_long],
         )
         self.w_uint64dtype = W_Dtype(
             types.UInt64(),
             alternate_constructors=[space.w_float],
             aliases=["float"],
         )
-
+        self.w_longlongdtype = W_Dtype(
+            types.Int64(),
+            num=9,
+            kind=SIGNEDLTR,
+            name='int64',
+            char='q',
+            w_box_type = space.gettypefor(interp_boxes.W_LongLongBox),
+            alternate_constructors=[space.w_long],
+        )
+        self.w_ulonglongdtype = W_Dtype(
+            types.UInt64(),
+            num=10,
+            kind=UNSIGNEDLTR,
+            name='uint64',
+            char='Q',
+            w_box_type = space.gettypefor(interp_boxes.W_ULongLongBox),
+        )
+        self.w_stringdtype = W_Dtype(
+            types.StringType(1),
+            num=18,
+            kind=STRINGLTR,
+            name='string',
+            char='S',
+            w_box_type = space.gettypefor(interp_boxes.W_StringBox),
+            alternate_constructors=[space.w_str],
+        )
+        self.w_unicodedtype = W_Dtype(
+            types.UnicodeType(1),
+            num=19,
+            kind=UNICODELTR,
+            name='unicode',
+            char='U',
+            w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
+            alternate_constructors=[space.w_unicode],
+        )
+        self.w_voiddtype = W_Dtype(
+            types.VoidType(0),
+            num=20,
+            kind=VOIDLTR,
+            name='void',
+            char='V',
+            w_box_type = space.gettypefor(interp_boxes.W_VoidBox),
+            #alternate_constructors=[space.w_buffer],
+            # XXX no buffer in space
+        )
         self.builtin_dtypes = [
             self.w_booldtype, self.w_int8dtype, self.w_uint8dtype,
             self.w_int16dtype, self.w_uint16dtype, self.w_int32dtype,
             self.w_uint32dtype, self.w_longdtype, self.w_ulongdtype,
-            self.w_int64dtype, self.w_uint64dtype, self.w_float32dtype,
-            self.w_float64dtype
+            self.w_longlongdtype, self.w_ulonglongdtype,
+            self.w_float32dtype,
+            self.w_float64dtype, self.w_stringdtype, self.w_unicodedtype,
+            self.w_voiddtype,
         ]
         self.dtypes_by_num_bytes = sorted(
             (dtype.itemtype.get_element_size(), dtype)
             for dtype in self.builtin_dtypes
         )
+        self.dtypes_by_name = {}
+        for dtype in self.builtin_dtypes:
+            self.dtypes_by_name[dtype.name] = dtype
+            can_name = dtype.kind + str(dtype.itemtype.get_element_size())
+            self.dtypes_by_name[can_name] = dtype
+            self.dtypes_by_name[byteorder_prefix + can_name] = dtype
+            self.dtypes_by_name['=' + can_name] = dtype
+            new_name = nonnative_byteorder_prefix + can_name
+            itemtypename = dtype.itemtype.__class__.__name__
+            itemtype = getattr(types, 'NonNative' + itemtypename)()
+            self.dtypes_by_name[new_name] = W_Dtype(
+                itemtype,
+                dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
+                native=False)
+            for alias in dtype.aliases:
+                self.dtypes_by_name[alias] = dtype
+            self.dtypes_by_name[dtype.char] = dtype
+
+        typeinfo_full = {
+            'LONGLONG': self.w_int64dtype,
+            'SHORT': self.w_int16dtype,
+            'VOID': self.w_voiddtype,
+            #'LONGDOUBLE':,
+            'UBYTE': self.w_uint8dtype,
+            'UINTP': self.w_ulongdtype,
+            'ULONG': self.w_ulongdtype,
+            'LONG': self.w_longdtype,
+            'UNICODE': self.w_unicodedtype,
+            #'OBJECT',
+            'ULONGLONG': self.w_ulonglongdtype,
+            'STRING': self.w_stringdtype,
+            #'CDOUBLE',
+            #'DATETIME',
+            'UINT': self.w_uint32dtype,
+            'INTP': self.w_longdtype,
+            #'HALF',
+            'BYTE': self.w_int8dtype,
+            #'CFLOAT': ,
+            #'TIMEDELTA',
+            'INT': self.w_int32dtype,
+            'DOUBLE': self.w_float64dtype,
+            'USHORT': self.w_uint16dtype,
+            'FLOAT': self.w_float32dtype,
+            'BOOL': self.w_booldtype,
+            #, 'CLONGDOUBLE']
+        }
+        typeinfo_partial = {
+            'Generic': interp_boxes.W_GenericBox,
+            'Character': interp_boxes.W_CharacterBox,
+            'Flexible': interp_boxes.W_FlexibleBox,
+            'Inexact': interp_boxes.W_InexactBox,
+            'Integer': interp_boxes.W_IntegerBox,
+            'SignedInteger': interp_boxes.W_SignedIntegerBox,
+            'UnsignedInteger': interp_boxes.W_UnsignedIntegerBox,
+            #'ComplexFloating',
+            'Number': interp_boxes.W_NumberBox,
+            'Floating': interp_boxes.W_FloatingBox
+        }
+        w_typeinfo = space.newdict()
+        for k, v in typeinfo_partial.iteritems():
+            space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v))
+        for k, dtype in typeinfo_full.iteritems():
+            itemsize = dtype.itemtype.get_element_size()
+            items_w = [space.wrap(dtype.char),
+                       space.wrap(dtype.num),
+                       space.wrap(itemsize * 8), # in case of changing
+                       # number of bits per byte in the future
+                       space.wrap(itemsize or 1)]
+            if dtype.is_int_type():
+                if dtype.kind == BOOLLTR:
+                    w_maxobj = space.wrap(1)
+                    w_minobj = space.wrap(0)
+                elif dtype.is_signed():
+                    w_maxobj = space.wrap(r_longlong((1 << (itemsize*8 - 1))
+                                          - 1))
+                    w_minobj = space.wrap(r_longlong(-1) << (itemsize*8 - 1))
+                else:
+                    w_maxobj = space.wrap(r_ulonglong(1 << (itemsize*8)) - 1)
+                    w_minobj = space.wrap(0)
+                items_w = items_w + [w_maxobj, w_minobj]
+            items_w = items_w + [dtype.w_box_type]
+                       
+            w_tuple = space.newtuple(items_w)
+            space.setitem(w_typeinfo, space.wrap(k), w_tuple)
+        self.w_typeinfo = w_typeinfo
 
 def get_dtype_cache(space):
     return space.fromcache(DtypeCache)

pypy/module/micronumpy/interp_iter.py

 from pypy.rlib import jit
 from pypy.rlib.objectmodel import instantiate
 from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
-     calculate_slice_strides, calculate_dot_strides