Anonymous avatar Anonymous committed 4fd8ce7 Merge

merged from pypy a3d4b51ec806

Comments (0)

Files changed (84)

py/_io/terminalwriter.py

                     ('srWindow', SMALL_RECT),
                     ('dwMaximumWindowSize', COORD)]
 
+    _GetStdHandle = ctypes.windll.kernel32.GetStdHandle
+    _GetStdHandle.argtypes = [wintypes.DWORD]
+    _GetStdHandle.restype = wintypes.HANDLE
     def GetStdHandle(kind):
-        return ctypes.windll.kernel32.GetStdHandle(kind)
+        return _GetStdHandle(kind)
 
-    SetConsoleTextAttribute = \
-        ctypes.windll.kernel32.SetConsoleTextAttribute
-
+    SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
+    SetConsoleTextAttribute.argtypes = [wintypes.HANDLE, wintypes.WORD]
+    SetConsoleTextAttribute.restype = wintypes.BOOL
+        
+    _GetConsoleScreenBufferInfo = \
+        ctypes.windll.kernel32.GetConsoleScreenBufferInfo
+    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE, 
+                                ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
+    _GetConsoleScreenBufferInfo.restype = wintypes.BOOL
     def GetConsoleInfo(handle):
         info = CONSOLE_SCREEN_BUFFER_INFO()
-        ctypes.windll.kernel32.GetConsoleScreenBufferInfo(\
-            handle, ctypes.byref(info))
+        _GetConsoleScreenBufferInfo(handle, ctypes.byref(info))
         return info
 
     def _getdimensions():

pypy/annotation/annrpython.py

         # make input arguments and set their type
         args_s = [self.typeannotation(t) for t in input_arg_types]
 
+        # XXX hack
+        annmodel.TLS.check_str_without_nul = (
+            self.translator.config.translation.check_str_without_nul)
+
         flowgraph, inputcells = self.get_call_parameters(function, args_s, policy)
         if not isinstance(flowgraph, FunctionGraph):
             assert isinstance(flowgraph, annmodel.SomeObject)

pypy/annotation/binaryop.py

 class __extend__(pairtype(SomeString, SomeString)):
 
     def union((str1, str2)):
-        return SomeString(can_be_None=str1.can_be_None or str2.can_be_None)
+        can_be_None = str1.can_be_None or str2.can_be_None
+        no_nul = str1.no_nul and str2.no_nul
+        return SomeString(can_be_None=can_be_None, no_nul=no_nul)
 
     def add((str1, str2)):
         # propagate const-ness to help getattr(obj, 'prefix' + const_name)
-        result = SomeString()
+        result = SomeString(no_nul=str1.no_nul and str2.no_nul)
         if str1.is_immutable_constant() and str2.is_immutable_constant():
             result.const = str1.const + str2.const
         return result
                 raise NotImplementedError(
                     "string formatting mixing strings and unicode not supported")
         getbookkeeper().count('strformat', str, s_tuple)
-        return SomeString()
+        no_nul = str.no_nul
+        for s_item in s_tuple.items:
+            if isinstance(s_item, SomeFloat):
+                pass   # or s_item is a subclass, like SomeInteger
+            elif isinstance(s_item, SomeString) and s_item.no_nul:
+                pass
+            else:
+                no_nul = False
+                break
+        return SomeString(no_nul=no_nul)
 
 
 class __extend__(pairtype(SomeString, SomeObject)):
     exec source.compile() in glob
 
 _make_none_union('SomeInstance',   'classdef=obj.classdef, can_be_None=True')
-_make_none_union('SomeString',      'can_be_None=True')
+_make_none_union('SomeString',      'no_nul=obj.no_nul, can_be_None=True')
 _make_none_union('SomeUnicodeString', 'can_be_None=True')
 _make_none_union('SomeList',         'obj.listdef')
 _make_none_union('SomeDict',          'obj.dictdef')

pypy/annotation/bookkeeper.py

             else:
                 raise Exception("seeing a prebuilt long (value %s)" % hex(x))
         elif issubclass(tp, str): # py.lib uses annotated str subclasses
+            no_nul = not '\x00' in x
             if len(x) == 1:
-                result = SomeChar()
+                result = SomeChar(no_nul=no_nul)
             else:
-                result = SomeString()
+                result = SomeString(no_nul=no_nul)
         elif tp is unicode:
             if len(x) == 1:
                 result = SomeUnicodeCodePoint()

pypy/annotation/listdef.py

             read_locations = self.read_locations.copy()
             other_read_locations = other.read_locations.copy()
             self.read_locations.update(other.read_locations)
-            self.patch()    # which should patch all refs to 'other'
             s_value = self.s_value
             s_other_value = other.s_value
             s_new_value = unionof(s_value, s_other_value)
+            if s_new_value != s_value:
+                if self.dont_change_any_more:
+                    raise TooLateForChange
             if isdegenerated(s_new_value):
                 if self.bookkeeper:
                     self.bookkeeper.ondegenerated(self, s_new_value)
                 elif other.bookkeeper:
                     other.bookkeeper.ondegenerated(other, s_new_value)
+            self.patch()    # which should patch all refs to 'other'
             if s_new_value != s_value:
-                if self.dont_change_any_more:
-                    raise TooLateForChange
                 self.s_value = s_new_value
                 # reflow from reading points
                 for position_key in read_locations:
 
 MOST_GENERAL_LISTDEF = ListDef(None, SomeObject())
 
-s_list_of_strings = SomeList(ListDef(None, SomeString(), resized = True))
+s_list_of_strings = SomeList(ListDef(None, SomeString(no_nul=True),
+                                     resized = True))

pypy/annotation/model.py

 DEBUG = False    # set to False to disable recording of debugging information
 
 class State(object):
-    pass
+    # A global attribute :-(  Patch it with 'True' to enable checking of
+    # the no_nul attribute...
+    check_str_without_nul = False
 TLS = State()
 
 class SomeObject(object):
     def __init__(self):
         pass
 
-class SomeString(SomeObject):
-    "Stands for an object which is known to be a string."
-    knowntype = str
+class SomeStringOrUnicode(SomeObject):
     immutable = True
-    def __init__(self, can_be_None=False):
-        self.can_be_None = can_be_None
+    can_be_None=False
+    no_nul = False  # No NUL character in the string.
+
+    def __init__(self, can_be_None=False, no_nul=False):
+        if can_be_None:
+            self.can_be_None = True
+        if no_nul:
+            self.no_nul = True
 
     def can_be_none(self):
         return self.can_be_None
 
+    def __eq__(self, other):
+        if self.__class__ is not other.__class__:
+            return False
+        d1 = self.__dict__
+        d2 = other.__dict__
+        if not TLS.check_str_without_nul:
+            d1 = d1.copy(); d1['no_nul'] = 0   # ignored
+            d2 = d2.copy(); d2['no_nul'] = 0   # ignored
+        return d1 == d2
+
+class SomeString(SomeStringOrUnicode):
+    "Stands for an object which is known to be a string."
+    knowntype = str
+
     def nonnoneify(self):
-        return SomeString(can_be_None=False)
+        return SomeString(can_be_None=False, no_nul=self.no_nul)
 
-class SomeUnicodeString(SomeObject):
+class SomeUnicodeString(SomeStringOrUnicode):
     "Stands for an object which is known to be an unicode string"
     knowntype = unicode
-    immutable = True
-    def __init__(self, can_be_None=False):
-        self.can_be_None = can_be_None
-
-    def can_be_none(self):
-        return self.can_be_None
 
     def nonnoneify(self):
-        return SomeUnicodeString(can_be_None=False)
+        return SomeUnicodeString(can_be_None=False, no_nul=self.no_nul)
 
 class SomeChar(SomeString):
     "Stands for an object known to be a string of length 1."
     can_be_None = False
-    def __init__(self):    # no 'can_be_None' argument here
-        pass
+    def __init__(self, no_nul=False):    # no 'can_be_None' argument here
+        if no_nul:
+            self.no_nul = True
 
 class SomeUnicodeCodePoint(SomeUnicodeString):
     "Stands for an object known to be a unicode codepoint."
     can_be_None = False
-    def __init__(self):    # no 'can_be_None' argument here
-        pass
+    def __init__(self, no_nul=False):    # no 'can_be_None' argument here
+        if no_nul:
+            self.no_nul = True
 
 SomeString.basestringclass = SomeString
 SomeString.basecharclass = SomeChar
 s_None = SomePBC([], can_be_None=True)
 s_Bool = SomeBool()
 s_ImpossibleValue = SomeImpossibleValue()
+s_Str0 = SomeString(no_nul=True)
 
 # ____________________________________________________________
 # weakrefs
 
 def not_const(s_obj):
     if s_obj.is_constant():
-        new_s_obj = SomeObject()
-        new_s_obj.__class__ = s_obj.__class__
+        new_s_obj = SomeObject.__new__(s_obj.__class__)
         dic = new_s_obj.__dict__ = s_obj.__dict__.copy()
         if 'const' in dic:
             del new_s_obj.const

pypy/annotation/test/test_annrpython.py

             return ''.join(g(n))
         s = a.build_types(f, [int])
         assert s.knowntype == str
+        assert s.no_nul
+
+    def test_str_split(self):
+        a = self.RPythonAnnotator()
+        def g(n):
+            if n:
+                return "test string"
+        def f(n):
+            if n:
+                return g(n).split(' ')
+        s = a.build_types(f, [int])
+        assert isinstance(s, annmodel.SomeList)
+        s_item = s.listdef.listitem.s_value
+        assert s_item.no_nul
 
     def test_str_splitlines(self):
         a = self.RPythonAnnotator()
         assert isinstance(s, annmodel.SomeList)
         assert s.listdef.listitem.resized
 
+    def test_str_strip(self):
+        a = self.RPythonAnnotator()
+        def f(n, a_str):
+            if n == 0:
+                return a_str.strip(' ')
+            elif n == 1:
+                return a_str.rstrip(' ')
+            else: 
+                return a_str.lstrip(' ')
+        s = a.build_types(f, [int, annmodel.SomeString(no_nul=True)])
+        assert s.no_nul
+
     def test_str_mul(self):
         a = self.RPythonAnnotator()
         def f(a_str):
             return obj.indirect()
         a = self.RPythonAnnotator()
         s = a.build_types(f, [bool])
-        assert s == annmodel.SomeString(can_be_None=True)
+        assert annmodel.SomeString(can_be_None=True).contains(s)
 
     def test_dont_see_AttributeError_clause(self):
         class Stuff:
         s = a.build_types(g, [int])
         assert not s.can_be_None
 
+    def test_string_noNUL_canbeNone(self):
+        def f(a):
+            if a:
+                return "abc"
+            else:
+                return None
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [int])
+        assert s.can_be_None
+        assert s.no_nul
+
+    def test_str_or_None(self):
+        def f(a):
+            if a:
+                return "abc"
+            else:
+                return None
+        def g(a):
+            x = f(a)
+            #assert x is not None
+            if x is None:
+                return "abcd"
+            return x
+            if isinstance(x, str):
+                return x
+            return "impossible"
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [int])
+        assert s.can_be_None
+        assert s.no_nul
+
     def test_emulated_pbc_call_simple(self):
         def f(a,b):
             return a + b
         assert isinstance(s, annmodel.SomeIterator)
         assert s.variant == ('items',)
 
+    def test_iteritems_str0(self):
+        def it(d):
+            return d.iteritems()
+        def f():
+            d0 = {'1a': '2a', '3': '4'}
+            for item in it(d0):
+                return "%s=%s" % item
+            raise ValueError
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [])
+        assert isinstance(s, annmodel.SomeString)
+        assert s.no_nul
+
     def test_non_none_and_none_with_isinstance(self):
         class A(object):
             pass

pypy/annotation/unaryop.py

         return SomeInteger(nonneg=True)
 
     def method_strip(str, chr):
-        return str.basestringclass()
+        return str.basestringclass(no_nul=str.no_nul)
 
     def method_lstrip(str, chr):
-        return str.basestringclass()
+        return str.basestringclass(no_nul=str.no_nul)
 
     def method_rstrip(str, chr):
-        return str.basestringclass()
+        return str.basestringclass(no_nul=str.no_nul)
 
     def method_join(str, s_list):
         if s_None.contains(s_list):
             if isinstance(str, SomeUnicodeString):
                 return immutablevalue(u"")
             return immutablevalue("")
-        return str.basestringclass()
+        no_nul = str.no_nul and s_item.no_nul
+        return str.basestringclass(no_nul=no_nul)
 
     def iter(str):
         return SomeIterator(str)
 
     def method_split(str, patt, max=-1):
         getbookkeeper().count("str_split", str, patt)
-        return getbookkeeper().newlist(str.basestringclass())
+        s_item = str.basestringclass(no_nul=str.no_nul)
+        return getbookkeeper().newlist(s_item)
 
     def method_rsplit(str, patt, max=-1):
         getbookkeeper().count("str_rsplit", str, patt)
-        return getbookkeeper().newlist(str.basestringclass())
+        s_item = str.basestringclass(no_nul=str.no_nul)
+        return getbookkeeper().newlist(s_item)
 
     def method_replace(str, s1, s2):
         return str.basestringclass()
 
     def getslice(str, s_start, s_stop):
         check_negative_slice(s_start, s_stop)
-        return str.basestringclass()
+        result = str.basestringclass(no_nul=str.no_nul)
+        return result
 
 class __extend__(SomeUnicodeString):
     def method_encode(uni, s_enc):

pypy/config/translationoption.py

                  default="off"),
     # jit_ffi is automatically turned on by withmod-_ffi (which is enabled by default)
     BoolOption("jit_ffi", "optimize libffi calls", default=False, cmdline=None),
+    BoolOption("check_str_without_nul",
+               "Forbid NUL chars in strings in some external function calls",
+               default=False, cmdline=None),
 
     # misc
     BoolOption("verbose", "Print extra information", default=False),

pypy/doc/Makefile

 	      "run these through (pdf)latex."
 
 man:
+	python config/generate.py
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	@echo
 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man"

pypy/doc/commandline_ref.rst

+Command line reference
+======================
+
+Manual pages
+------------
+
+.. toctree::
+   :maxdepth: 1
+
+   man/pypy.1.rst
 # built documents.
 #
 # The short X.Y version.
-version = '1.7'
+version = '1.8'
 # The full version, including alpha/beta/rc tags.
-release = '1.7'
+release = '1.8'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pypy/doc/config/translation.check_str_without_nul.txt

+If turned on, the annotator will keep track of which strings can
+potentially contain NUL characters, and complain if one such string
+is passed to some external functions --- e.g. if it is used as a
+filename in os.open().  Defaults to False because it is usually more
+pain than benefit, but turned on by targetpypystandalone.

pypy/doc/config/translation.log.txt

 
 These must be enabled by setting the PYPYLOG environment variable.
 The exact set of features supported by PYPYLOG is described in
-pypy/translation/c/src/debug.h.
+pypy/translation/c/src/debug_print.h.

pypy/doc/garbage_collection.rst

 So as a first approximation, when compared to the Hybrid GC, the
 Minimark GC saves one word of memory per old object.
 
-There are a number of environment variables that can be tweaked to
-influence the GC.  (Their default value should be ok for most usages.)
-You can read more about them at the start of
-`pypy/rpython/memory/gc/minimark.py`_.
+There are :ref:`a number of environment variables
+<minimark-environment-variables>` that can be tweaked to influence the
+GC.  (Their default value should be ok for most usages.)
 
 In more detail:
 
   are preserved.  If the object dies then the pre-reserved location
   becomes free garbage, to be collected at the next major collection.
 
-
 .. include:: _ref.txt

pypy/doc/gc_info.rst

+Garbage collector configuration
+===============================
+
+.. _minimark-environment-variables:
+
+Minimark
+--------
+
+PyPy's default ``minimark`` garbage collector is configurable through
+several environment variables:
+
+``PYPY_GC_NURSERY``
+    The nursery size.
+    Defaults to ``4MB``.
+    Small values (like 1 or 1KB) are useful for debugging.
+
+``PYPY_GC_MAJOR_COLLECT``
+    Major collection memory factor.
+    Default is ``1.82``, which means trigger a major collection when the
+    memory consumed equals 1.82 times the memory really used at the end
+    of the previous major collection.
+
+``PYPY_GC_GROWTH``
+    Major collection threshold's max growth rate.
+    Default is ``1.4``.
+    Useful to collect more often than normally on sudden memory growth,
+    e.g. when there is a temporary peak in memory usage.
+
+``PYPY_GC_MAX``
+    The max heap size.
+    If coming near this limit, it will first collect more often, then
+    raise an RPython MemoryError, and if that is not enough, crash the
+    program with a fatal error.
+    Try values like ``1.6GB``.
+
+``PYPY_GC_MAX_DELTA``
+    The major collection threshold will never be set to more than
+    ``PYPY_GC_MAX_DELTA`` the amount really used after a collection.
+    Defaults to 1/8th of the total RAM size (which is constrained to be
+    at most 2/3/4GB on 32-bit systems).
+    Try values like ``200MB``.
+
+``PYPY_GC_MIN``
+    Don't collect while the memory size is below this limit.
+    Useful to avoid spending all the time in the GC in very small
+    programs.
+    Defaults to 8 times the nursery.
+
+``PYPY_GC_DEBUG``
+    Enable extra checks around collections that are too slow for normal
+    use.
+    Values are ``0`` (off), ``1`` (on major collections) or ``2`` (also
+    on minor collections).

pypy/doc/index.rst

    getting-started-dev.rst
    windows.rst
    faq.rst
+   commandline_ref.rst
    architecture.rst
    coding-guide.rst
    cpython_differences.rst
    garbage_collection.rst
+   gc_info.rst
    interpreter.rst
    objspace.rst
    __pypy__-module.rst

pypy/doc/man/pypy.1.rst

 -S
     Do not ``import site`` on initialization.
 
+-s
+    Don't add the user site directory to `sys.path`.
+
 -u
     Unbuffered binary ``stdout`` and ``stderr``.
 
 -E
     Ignore environment variables (such as ``PYTHONPATH``).
 
+-B
+    Disable writing bytecode (``.pyc``) files.
+
 --version
     Print the PyPy version.
 
         Optimizations to enabled or ``all``.
         Warning, this option is dangerous, and should be avoided.
 
+ENVIRONMENT
+===========
+
+``PYTHONPATH``
+    Add directories to pypy's module search path.
+    The format is the same as shell's ``PATH``.
+
+``PYTHONSTARTUP``
+    A script referenced by this variable will be executed before the
+    first prompt is displayed, in interactive mode.
+
+``PYTHONDONTWRITEBYTECODE``
+    If set to a non-empty value, equivalent to the ``-B`` option.
+    Disable writing ``.pyc`` files.
+
+``PYTHONINSPECT``
+    If set to a non-empty value, equivalent to the ``-i`` option.
+    Inspect interactively after running the specified script.
+
+``PYTHONIOENCODING``
+    If this is set, it overrides the encoding used for
+    *stdin*/*stdout*/*stderr*.
+    The syntax is *encodingname*:*errorhandler*
+    The *errorhandler* part is optional and has the same meaning as in
+    `str.encode`.
+
+``PYTHONNOUSERSITE``
+    If set to a non-empty value, equivalent to the ``-s`` option.
+    Don't add the user site directory to `sys.path`.
+
+``PYTHONWARNINGS``
+    If set, equivalent to the ``-W`` option (warning control).
+    The value should be a comma-separated list of ``-W`` parameters.
+
+``PYPYLOG``
+    If set to a non-empty value, enable logging, the format is:
+
+    *fname*
+        logging for profiling: includes all
+        ``debug_start``/``debug_stop`` but not any nested
+        ``debug_print``.
+        *fname* can be ``-`` to log to *stderr*.
+
+    ``:``\ *fname*
+        Full logging, including ``debug_print``.
+
+    *prefix*\ ``:``\ *fname*
+        Conditional logging.
+        Multiple prefixes can be specified, comma-separated.
+        Only sections whose name match the prefix will be logged.
+
+    ``PYPYLOG``\ =\ ``jit-log-opt,jit-backend:``\ *logfile* will
+    generate a log suitable for *jitviewer*, a tool for debugging
+    performance issues under PyPy.
+
+.. include:: ../gc_info.rst
+   :start-line: 7
+
 SEE ALSO
 ========
 

pypy/doc/release-1.8.0.rst

+============================
+PyPy 1.8 - business as usual
+============================
+
+We're pleased to announce the 1.8 release of PyPy. As became a habit, this
+release brings a lot of bugfixes, performance and memory improvements over
+the 1.7 release. The main highlight of the release is the introduction of
+list strategies which makes homogenous lists more efficient both in terms
+of performance and memory. Otherwise it's "business as usual" in the sense
+that performance improved roughly 10% on average since the previous release.
+You can download the PyPy 1.8 release here:
+
+    http://pypy.org/download.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 1.8 and cpython 2.7.1`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 32/64 or
+Windows 32. Windows 64 work is ongoing, but not yet natively supported.
+
+.. _`pypy 1.8 and cpython 2.7.1`: http://speed.pypy.org
+
+
+Highlights
+==========
+
+* List strategies. Now lists that contain only ints or only floats should
+  be as efficient as storing them in a binary-packed array. It also improves
+  the JIT performance in places that use such lists. There are also special
+  strategies for unicode and string lists.
+
+* As usual, numerous performance improvements. There are too many examples
+  of python constructs that now should behave faster to list them.
+
+* Bugfixes and compatibility fixes with CPython.
+
+* Windows fixes.
+
+* NumPy effort progress; for the exact list of things that have been done,
+  consult the `numpy status page`_. A tentative list of things that has
+  been done:
+
+  xxxx # list it, multidim arrays in particular
+
+* Fundraising XXX
+
+.. _`numpy status page`: xxx
+.. _`numpy status update blog report`: xxx

pypy/interpreter/astcompiler/optimize.py

                     # narrow builds will return a surrogate.  In both
                     # the cases skip the optimization in order to
                     # produce compatible pycs.
-                    if (self.space.isinstance_w(w_obj, self.space.w_unicode)
-                        and 
+                    if (self.space.isinstance_w(w_obj, self.space.w_unicode) and
                         self.space.isinstance_w(w_const, self.space.w_unicode)):
                         unistr = self.space.unicode_w(w_const)
                         if len(unistr) == 1:
                         else:
                             ch = 0
                         if (ch > 0xFFFF or
-                            (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFFF)):
+                            (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
                             return subs
 
                     return ast.Const(w_const, subs.lineno, subs.col_offset)

pypy/interpreter/astcompiler/test/test_compiler.py

         # Just checking this doesn't crash out
         self.count_instructions(source)
 
-    def test_const_fold_unicode_subscr(self):
+    def test_const_fold_unicode_subscr(self, monkeypatch):
         source = """def f():
         return u"abc"[0]
         """
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
                           ops.RETURN_VALUE: 1}
 
+        monkeypatch.setattr(optimize, "MAXUNICODE", 0xFFFF)
+        source = """def f():
+        return u"\uE01F"[0]
+        """
+        counts = self.count_instructions(source)
+        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+        monkeypatch.undo()
+
         # getslice is not yet optimized.
         # Still, check a case which yields the empty string.
         source = """def f():

pypy/interpreter/baseobjspace.py

     def str_w(self, w_obj):
         return w_obj.str_w(self)
 
+    def str0_w(self, w_obj):
+        "Like str_w, but rejects strings with NUL bytes."
+        from pypy.rlib import rstring
+        result = w_obj.str_w(self)
+        if '\x00' in result:
+            raise OperationError(self.w_TypeError, self.wrap(
+                    'argument must be a string without NUL characters'))
+        return rstring.assert_str0(result)
+
     def int_w(self, w_obj):
         return w_obj.int_w(self)
 
     def unicode_w(self, w_obj):
         return w_obj.unicode_w(self)
 
+    def unicode0_w(self, w_obj):
+        "Like unicode_w, but rejects strings with NUL bytes."
+        from pypy.rlib import rstring
+        result = w_obj.unicode_w(self)
+        if u'\x00' in result:
+            raise OperationError(self.w_TypeError, self.wrap(
+                    'argument must be a unicode string without NUL characters'))
+        return rstring.assert_str0(result)
+
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.
     'UnicodeEncodeError',
     'UnicodeDecodeError',
     ]
+    
+if sys.platform.startswith("win"):
+    ObjSpace.ExceptionTable += ['WindowsError']
 
 ## Irregular part of the interface:
 #

pypy/interpreter/gateway.py

     def visit_str_or_None(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
+    def visit_str0(self, el, app_sig):
+        self.checked_space_method(el, app_sig)
+
     def visit_nonnegint(self, el, app_sig):
         self.checked_space_method(el, app_sig)
 
     def visit_str_or_None(self, typ):
         self.run_args.append("space.str_or_None_w(%s)" % (self.scopenext(),))
 
+    def visit_str0(self, typ):
+        self.run_args.append("space.str0_w(%s)" % (self.scopenext(),))
+
     def visit_nonnegint(self, typ):
         self.run_args.append("space.gateway_nonnegint_w(%s)" % (
             self.scopenext(),))
     def visit_str_or_None(self, typ):
         self.unwrap.append("space.str_or_None_w(%s)" % (self.nextarg(),))
 
+    def visit_str0(self, typ):
+        self.unwrap.append("space.str0_w(%s)" % (self.nextarg(),))
+
     def visit_nonnegint(self, typ):
         self.unwrap.append("space.gateway_nonnegint_w(%s)" % (self.nextarg(),))
 

pypy/interpreter/mixedmodule.py

             space.call_method(self.w_dict, 'update', self.w_initialdict)
 
         for w_submodule in self.submodules_w:
-            name = space.str_w(w_submodule.w_name)
+            name = space.str0_w(w_submodule.w_name)
             space.setitem(self.w_dict, space.wrap(name.split(".")[-1]), w_submodule)
             space.getbuiltinmodule(name)
 

pypy/interpreter/module.py

     def install(self):
         """NOT_RPYTHON: installs this module into space.builtin_modules"""
         w_mod = self.space.wrap(self)
-        self.space.builtin_modules[self.space.unwrap(self.w_name)] = w_mod
+        modulename = self.space.str0_w(self.w_name)
+        self.space.builtin_modules[modulename] = w_mod
 
     def setup_after_space_initialization(self):
         """NOT_RPYTHON: to allow built-in modules to do some more setup

pypy/interpreter/test/test_objspace.py

         res = self.space.interp_w(Function, w(None), can_be_None=True)
         assert res is None
 
+    def test_str0_w(self):
+        space = self.space
+        w = space.wrap
+        assert space.str0_w(w("123")) == "123"
+        exc = space.raises_w(space.w_TypeError, space.str0_w, w("123\x004"))
+        assert space.unicode0_w(w(u"123")) == u"123"
+        exc = space.raises_w(space.w_TypeError, space.unicode0_w, w(u"123\x004"))
+
     def test_getindex_w(self):
         w_instance1 = self.space.appexec([], """():
             class X(object):

pypy/jit/backend/llgraph/llimpl.py

         self.overflow_flag = ovf
         return z
 
+    def op_keepalive(self, _, x):
+        pass
+
     # ----------
     # delegating to the builtins do_xxx() (done automatically for simple cases)
 

pypy/jit/backend/x86/regalloc.py

         if jump_op is not None and jump_op.getdescr() is descr:
             self._compute_hint_frame_locations_from_descr(descr)
 
+    def consider_keepalive(self, op):
+        pass
+
     def not_implemented_op(self, op):
         not_implemented("not implemented operation: %s" % op.getopname())
 

pypy/jit/codewriter/flatten.py

         if len(block.exits) == 1:
             # A single link, fall-through
             link = block.exits[0]
-            assert link.exitcase is None
+            assert link.exitcase in (None, False, True)
+            # the cases False or True should not really occur, but can show
+            # up in the manually hacked graphs for generators...
             self.make_link(link)
         #
         elif block.exitswitch is c_last_exception:

pypy/jit/codewriter/policy.py

         mod = func.__module__ or '?'
         if mod.startswith('pypy.rpython.module.'):
             return True
-        if mod.startswith('pypy.translator.'): # XXX wtf?
+        if mod == 'pypy.translator.goal.nanos':    # more helpers
             return True
         return False
 

pypy/jit/metainterp/executor.py

         assert isinstance(x, r_longlong)  # 32-bit
         return BoxFloat(x)
 
+def do_keepalive(cpu, _, x):
+    pass
+
 # ____________________________________________________________
 
 ##def do_force_token(cpu):

pypy/jit/metainterp/pyjitpl.py

         any_operation = len(self.metainterp.history.operations) > 0
         jitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
         self.verify_green_args(jitdriver_sd, greenboxes)
-        self.debug_merge_point(jitdriver_sd, jdindex, self.metainterp.in_recursion,
+        self.debug_merge_point(jitdriver_sd, jdindex, self.metainterp.portal_call_depth,
                                greenboxes)
         
         if self.metainterp.seen_loop_header_for_jdindex < 0:
             if not any_operation:
                 return
-            if self.metainterp.in_recursion or not self.metainterp.get_procedure_token(greenboxes, True):
+            if self.metainterp.portal_call_depth or not self.metainterp.get_procedure_token(greenboxes, True):
                 if not jitdriver_sd.no_loop_header:
                     return
             # automatically add a loop_header if there is none
         self.metainterp.seen_loop_header_for_jdindex = -1
 
         #
-        if not self.metainterp.in_recursion:
+        if not self.metainterp.portal_call_depth:
             assert jitdriver_sd is self.metainterp.jitdriver_sd
             # Set self.pc to point to jit_merge_point instead of just after:
             # if reached_loop_header() raises SwitchToBlackhole, then the
                                     assembler_call=True)
             raise ChangeFrame
 
-    def debug_merge_point(self, jitdriver_sd, jd_index, in_recursion, greenkey):
+    def debug_merge_point(self, jitdriver_sd, jd_index, portal_call_depth, greenkey):
         # debugging: produce a DEBUG_MERGE_POINT operation
         loc = jitdriver_sd.warmstate.get_location_str(greenkey)
         debug_print(loc)
-        args = [ConstInt(jd_index), ConstInt(in_recursion)] + greenkey
+        args = [ConstInt(jd_index), ConstInt(portal_call_depth)] + greenkey
         self.metainterp.history.record(rop.DEBUG_MERGE_POINT, args, None)
 
     @arguments("box", "label")
             resbox = self.metainterp.execute_and_record_varargs(
                 rop.CALL_MAY_FORCE, allboxes, descr=descr)
             self.metainterp.vrefs_after_residual_call()
+            vablebox = None
             if assembler_call:
-                self.metainterp.direct_assembler_call(assembler_call_jd)
+                vablebox = self.metainterp.direct_assembler_call(
+                    assembler_call_jd)
             if resbox is not None:
                 self.make_result_of_lastop(resbox)
             self.metainterp.vable_after_residual_call()
             self.generate_guard(rop.GUARD_NOT_FORCED, None)
+            if vablebox is not None:
+                self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
             self.metainterp.handle_possible_exception()
             return resbox
         else:
 # ____________________________________________________________
 
 class MetaInterp(object):
-    in_recursion = 0
+    portal_call_depth = 0
     cancel_count = 0
 
     def __init__(self, staticdata, jitdriver_sd):
 
     def newframe(self, jitcode, greenkey=None):
         if jitcode.is_portal:
-            self.in_recursion += 1
+            self.portal_call_depth += 1
         if greenkey is not None and self.is_main_jitcode(jitcode):
             self.portal_trace_positions.append(
                     (greenkey, len(self.history.operations)))
         frame = self.framestack.pop()
         jitcode = frame.jitcode
         if jitcode.is_portal:
-            self.in_recursion -= 1
+            self.portal_call_depth -= 1
         if frame.greenkey is not None and self.is_main_jitcode(jitcode):
             self.portal_trace_positions.append(
                     (None, len(self.history.operations)))
         raise self.staticdata.ExitFrameWithExceptionRef(self.cpu, excvaluebox.getref_base())
 
     def check_recursion_invariant(self):
-        in_recursion = -1
+        portal_call_depth = -1
         for frame in self.framestack:
             jitcode = frame.jitcode
             assert jitcode.is_portal == len([
                 jd for jd in self.staticdata.jitdrivers_sd
                    if jd.mainjitcode is jitcode])
             if jitcode.is_portal:
-                in_recursion += 1
-        if in_recursion != self.in_recursion:
-            print "in_recursion problem!!!"
-            print in_recursion, self.in_recursion
+                portal_call_depth += 1
+        if portal_call_depth != self.portal_call_depth:
+            print "portal_call_depth problem!!!"
+            print portal_call_depth, self.portal_call_depth
             for frame in self.framestack:
                 jitcode = frame.jitcode
                 if jitcode.is_portal:
 
     def initialize_state_from_start(self, original_boxes):
         # ----- make a new frame -----
-        self.in_recursion = -1 # always one portal around
+        self.portal_call_depth = -1 # always one portal around
         self.framestack = []
         f = self.newframe(self.jitdriver_sd.mainjitcode)
         f.setup_call(original_boxes)
-        assert self.in_recursion == 0
+        assert self.portal_call_depth == 0
         self.virtualref_boxes = []
         self.initialize_withgreenfields(original_boxes)
         self.initialize_virtualizable(original_boxes)
         # otherwise the jit_virtual_refs are left in a dangling state.
         rstack._stack_criticalcode_start()
         try:
-            self.in_recursion = -1 # always one portal around
+            self.portal_call_depth = -1 # always one portal around
             self.history = history.History()
             inputargs_and_holes = self.rebuild_state_after_failure(resumedescr)
             self.history.inputargs = [box for box in inputargs_and_holes if box]
         token = warmrunnerstate.get_assembler_token(greenargs)
         op = op.copy_and_change(rop.CALL_ASSEMBLER, args=args, descr=token)
         self.history.operations.append(op)
+        #
+        # To fix an obscure issue, make sure the vable stays alive
+        # longer than the CALL_ASSEMBLER operation.  We do it by
+        # inserting explicitly an extra KEEPALIVE operation.
+        jd = token.outermost_jitdriver_sd
+        if jd.index_of_virtualizable >= 0:
+            return args[jd.index_of_virtualizable]
+        else:
+            return None
 
 # ____________________________________________________________
 

pypy/jit/metainterp/resoperation.py

     'COPYUNICODECONTENT/5',
     'QUASIIMMUT_FIELD/1d',    # [objptr], descr=SlowMutateDescr
     'RECORD_KNOWN_CLASS/2',   # [objptr, clsptr]
+    'KEEPALIVE/1',
 
     '_CANRAISE_FIRST', # ----- start of can_raise operations -----
     '_CALL_FIRST',

pypy/jit/metainterp/test/test_ajit.py

         res = self.interp_operations(f, [42])
         assert res == ord(u"?")
 
+    def test_char_in_constant_string(self):
+        def g(string):
+            return '\x00' in string
+        def f():
+            if g('abcdef'): return -60
+            if not g('abc\x00ef'): return -61
+            return 42
+        res = self.interp_operations(f, [])
+        assert res == 42
+        self.check_operations_history({'finish': 1})   # nothing else
+
     def test_residual_call(self):
         @dont_look_inside
         def externfn(x, y):
         # here it works again
         self.check_operations_history(guard_class=0, record_known_class=1)
 
+    def test_generator(self):
+        def g(n):
+            yield n+1
+            yield n+2
+            yield n+3
+        def f(n):
+            gen = g(n)
+            return gen.next() * gen.next() * gen.next()
+        res = self.interp_operations(f, [10])
+        assert res == 11 * 12 * 13
+        self.check_operations_history(int_add=3, int_mul=2)
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):

pypy/module/_ffi/test/test__ffi.py

 
     def test_convert_strings_to_char_p(self):
         """
+            DLLEXPORT
             long mystrlen(char* s)
             {
                 long len = 0;
     def test_convert_unicode_to_unichar_p(self):
         """
             #include <wchar.h>
+            DLLEXPORT
             long mystrlen_u(wchar_t* s)
             {
                 long len = 0;
 
     def test_keepalive_temp_buffer(self):
         """
+            DLLEXPORT
             char* do_nothing(char* s)
             {
                 return s;
         from _ffi import CDLL, types
         libfoo = CDLL(self.libfoo_name)
         raises(AttributeError, "libfoo.getfunc('I_do_not_exist', [], types.void)")
+        if self.iswin32:
+            skip("unix specific")
         libnone = CDLL(None)
         raises(AttributeError, "libnone.getfunc('I_do_not_exist', [], types.void)")

pypy/module/_file/test/test_file.py

 
         if option.runappdirect:
             py.test.skip("works with internals of _file impl on py.py")
+        import platform
+        if platform.system() == 'Windows':
+            # XXX This test crashes until someone implements something like
+            # XXX verify_fd from
+            # XXX http://hg.python.org/cpython/file/80ddbd822227/Modules/posixmodule.c#l434
+            # XXX and adds it to fopen
+            assert False
 
         state = [0]
         def read(fd, n=None):

pypy/module/bz2/interp_bz2.py

     if basemode == "a":
         raise OperationError(space.w_ValueError,
                              space.wrap("cannot append to bz2 file"))
-    stream = open_path_helper(space.str_w(w_path), os_flags, False)
+    stream = open_path_helper(space.str0_w(w_path), os_flags, False)
     if reading:
         bz2stream = ReadBZ2Filter(space, stream, buffering)
         buffering = 0     # by construction, the ReadBZ2Filter acts like

pypy/module/cpyext/include/pythonrun.h

 
 #define Py_FrozenFlag 0
 #define Py_VerboseFlag 0
+#define Py_DebugFlag 1
 
 typedef struct {
     int cf_flags;  /* bitmask of CO_xxx flags relevant to future */

pypy/module/gc/interp_gc.py

 
 # ____________________________________________________________
 
-@unwrap_spec(filename=str)
+@unwrap_spec(filename='str0')
 def dump_heap_stats(space, filename):
     tb = rgc._heap_stats()
     if not tb:

pypy/module/imp/importing.py

     ctxt_package = None
     if ctxt_w_package is not None and ctxt_w_package is not space.w_None:
         try:
-            ctxt_package = space.str_w(ctxt_w_package)
+            ctxt_package = space.str0_w(ctxt_w_package)
         except OperationError, e:
             if not e.match(space, space.w_TypeError):
                 raise
         ctxt_name = None
         if ctxt_w_name is not None:
             try:
-                ctxt_name = space.str_w(ctxt_w_name)
+                ctxt_name = space.str0_w(ctxt_w_name)
             except OperationError, e:
                 if not e.match(space, space.w_TypeError):
                     raise
     return rel_modulename, rel_level
 
 
-@unwrap_spec(name=str, level=int)
+@unwrap_spec(name='str0', level=int)
 def importhook(space, name, w_globals=None,
                w_locals=None, w_fromlist=None, level=-1):
     modulename = name
                     fromlist_w = space.fixedview(w_all)
             for w_name in fromlist_w:
                 if try_getattr(space, w_mod, w_name) is None:
-                    load_part(space, w_path, prefix, space.str_w(w_name), w_mod,
-                              tentative=1)
+                    load_part(space, w_path, prefix, space.str0_w(w_name),
+                              w_mod, tentative=1)
         return w_mod
     else:
         return first
     def __init__(self, space):
         pass
 
-    @unwrap_spec(path=str)
+    @unwrap_spec(path='str0')
     def descr_init(self, space, path):
         if not path:
             raise OperationError(space.w_ImportError, space.wrap(
                 if w_loader:
                     return FindInfo.fromLoader(w_loader)
 
-            path = space.str_w(w_pathitem)
+            path = space.str0_w(w_pathitem)
             filepart = os.path.join(path, partname)
             if os.path.isdir(filepart) and case_ok(filepart):
                 initfile = os.path.join(filepart, '__init__')
             space.wrap("reload() argument must be module"))
 
     w_modulename = space.getattr(w_module, space.wrap("__name__"))
-    modulename = space.str_w(w_modulename)
+    modulename = space.str0_w(w_modulename)
     if not space.is_w(check_sys_modules(space, w_modulename), w_module):
         raise operationerrfmt(
             space.w_ImportError,

pypy/module/imp/interp_imp.py

         return space.interp_w(W_File, w_file).stream
 
 def find_module(space, w_name, w_path=None):
-    name = space.str_w(w_name)
+    name = space.str0_w(w_name)
     if space.is_w(w_path, space.w_None):
         w_path = None
 
 def load_module(space, w_name, w_file, w_filename, w_info):
     w_suffix, w_filemode, w_modtype = space.unpackiterable(w_info)
 
-    filename = space.str_w(w_filename)
+    filename = space.str0_w(w_filename)
     filemode = space.str_w(w_filemode)
     if space.is_w(w_file, space.w_None):
         stream = None
         space, w_name, find_info, reuse=True)
 
 def load_source(space, w_modulename, w_filename, w_file=None):
-    filename = space.str_w(w_filename)
+    filename = space.str0_w(w_filename)
 
     stream = get_file(space, w_file, filename, 'U')
 
         stream.close()
     return w_mod
 
-@unwrap_spec(filename=str)
+@unwrap_spec(filename='str0')
 def _run_compiled_module(space, w_modulename, filename, w_file, w_module):
     # the function 'imp._run_compiled_module' is a pypy-only extension
     stream = get_file(space, w_file, filename, 'rb')
     if space.is_w(w_file, space.w_None):
         stream.close()
 
-@unwrap_spec(filename=str)
+@unwrap_spec(filename='str0')
 def load_compiled(space, w_modulename, filename, w_file=None):
     w_mod = space.wrap(Module(space, w_modulename))
     importing._prepare_module(space, w_mod, filename, None)
     return space.wrap(Module(space, w_name, add_package=False))
 
 def init_builtin(space, w_name):
-    name = space.str_w(w_name)
+    name = space.str0_w(w_name)
     if name not in space.builtin_modules:
         return
     if space.finditem(space.sys.get('modules'), w_name) is not None:
     return None
 
 def is_builtin(space, w_name):
-    name = space.str_w(w_name)
+    name = space.str0_w(w_name)
     if name not in space.builtin_modules:
         return space.wrap(0)
     if space.finditem(space.sys.get('modules'), w_name) is not None:

pypy/module/micronumpy/__init__.py

         ('bitwise_not', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
+        ('logical_and', 'logical_and'),
+        ('logical_xor', 'logical_xor'),
+        ('logical_not', 'logical_not'),
+        ('logical_or', 'logical_or'),
     ]:
         interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl
 

pypy/module/micronumpy/interp_boxes.py

     descr_sub = _binop_impl("subtract")
     descr_mul = _binop_impl("multiply")
     descr_div = _binop_impl("divide")
+    descr_truediv = _binop_impl("true_divide")
     descr_pow = _binop_impl("power")
     descr_eq = _binop_impl("equal")
     descr_ne = _binop_impl("not_equal")
     __sub__ = interp2app(W_GenericBox.descr_sub),
     __mul__ = interp2app(W_GenericBox.descr_mul),
     __div__ = interp2app(W_GenericBox.descr_div),
+    __truediv__ = interp2app(W_GenericBox.descr_truediv),
     __pow__ = interp2app(W_GenericBox.descr_pow),
 
     __radd__ = interp2app(W_GenericBox.descr_radd),

pypy/module/micronumpy/interp_iter.py

 
     def apply_transformations(self, arr, transformations):
         v = self
-        for transform in transformations:
-            v = v.transform(arr, transform)
+        if transformations is not None:
+            for transform in transformations:
+                v = v.transform(arr, transform)
         return v
 
     def transform(self, arr, t):

pypy/module/micronumpy/interp_numarray.py

 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import (interp_ufuncs, interp_dtype, interp_boxes,
-    signature, support)
+    signature, support, loop)
 from pypy.module.micronumpy.strides import (calculate_slice_strides,
     shape_agreement, find_shape_and_elems, get_shape_from_iterable,
     calc_new_strides, to_coords)
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rlib.rstring import StringBuilder
-from pypy.module.micronumpy.interp_iter import (ArrayIterator, OneDimIterator,
+from pypy.module.micronumpy.interp_iter import (ArrayIterator,
     SkipLastAxisIterator, Chunk, ViewIterator)
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 
 
-numpy_driver = jit.JitDriver(
-    greens=['shapelen', 'sig'],
-    virtualizables=['frame'],
-    reds=['result_size', 'frame', 'ri', 'self', 'result'],
-    get_printable_location=signature.new_printable_location('numpy'),
-    name='numpy',
-)
-all_driver = jit.JitDriver(
-    greens=['shapelen', 'sig'],
-    virtualizables=['frame'],
-    reds=['frame', 'self', 'dtype'],
-    get_printable_location=signature.new_printable_location('all'),
-    name='numpy_all',
-)
-any_driver = jit.JitDriver(
-    greens=['shapelen', 'sig'],
-    virtualizables=['frame'],
-    reds=['frame', 'self', 'dtype'],
-    get_printable_location=signature.new_printable_location('any'),
-    name='numpy_any',
-)
-slice_driver = jit.JitDriver(
-    greens=['shapelen', 'sig'],
-    virtualizables=['frame'],
-    reds=['self', 'frame', 'arr'],
-    get_printable_location=signature.new_printable_location('slice'),
-    name='numpy_slice',
-)
 count_driver = jit.JitDriver(
     greens=['shapelen'],
     virtualizables=['frame'],
     descr_prod = _reduce_ufunc_impl("multiply", True)
     descr_max = _reduce_ufunc_impl("maximum")
     descr_min = _reduce_ufunc_impl("minimum")
+    descr_all = _reduce_ufunc_impl('logical_and')
+    descr_any = _reduce_ufunc_impl('logical_or')
 
     def _reduce_argmax_argmin_impl(op_name):
         reduce_driver = jit.JitDriver(
             return space.wrap(loop(self))
         return func_with_new_name(impl, "reduce_arg%s_impl" % op_name)
 
-    def _all(self):
-        dtype = self.find_dtype()
-        sig = self.find_sig()
-        frame = sig.create_frame(self)
-        shapelen = len(self.shape)
-        while not frame.done():
-            all_driver.jit_merge_point(sig=sig,
-                                       shapelen=shapelen, self=self,
-                                       dtype=dtype, frame=frame)
-            if not dtype.itemtype.bool(sig.eval(frame, self)):
-                return False
-            frame.next(shapelen)
-        return True
-
-    def descr_all(self, space):
-        return space.wrap(self._all())
-
-    def _any(self):
-        dtype = self.find_dtype()
-        sig = self.find_sig()
-        frame = sig.create_frame(self)
-        shapelen = len(self.shape)
-        while not frame.done():
-            any_driver.jit_merge_point(sig=sig, frame=frame,
-                                       shapelen=shapelen, self=self,
-                                       dtype=dtype)
-            if dtype.itemtype.bool(sig.eval(frame, self)):
-                return True
-            frame.next(shapelen)
-        return False
-
-    def descr_any(self, space):
-        return space.wrap(self._any())
-
     descr_argmax = _reduce_argmax_argmin_impl("max")
     descr_argmin = _reduce_argmax_argmin_impl("min")
 
         out_size = support.product(out_shape)
         result = W_NDimArray(out_size, out_shape, dtype)
         # This is the place to add fpypy and blas
-        return multidim_dot(space, self.get_concrete(), 
+        return multidim_dot(space, self.get_concrete(),
                             other.get_concrete(), result, dtype,
                             other_critical_dim)
 
     def descr_get_ndim(self, space):
         return space.wrap(len(self.shape))
 
+    def descr_get_itemsize(self, space):
+        return space.wrap(self.find_dtype().itemtype.get_element_size())
+
+    def descr_get_nbytes(self, space):
+        return space.wrap(self.size * self.find_dtype().itemtype.get_element_size())
+
     @jit.unroll_safe
     def descr_get_shape(self, space):
         return space.newtuple([space.wrap(i) for i in self.shape])
             w_shape = space.newtuple(args_w)
         new_shape = get_shape_from_iterable(space, self.size, w_shape)
         return self.reshape(space, new_shape)
-        
+
     def reshape(self, space, new_shape):
         concrete = self.get_concrete()
         # Since we got to here, prod(new_shape) == self.size
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "non-int arg not supported"))
 
+    def compute_first_step(self, sig, frame):
+        pass
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
         raise NotImplementedError
 
     def compute(self):
-        result = W_NDimArray(self.size, self.shape, self.find_dtype())
-        shapelen = len(self.shape)
-        sig = self.find_sig()
-        frame = sig.create_frame(self)
-        ri = ArrayIterator(self.size)
-        while not ri.done():
-            numpy_driver.jit_merge_point(sig=sig,
-                                         shapelen=shapelen,
-                                         result_size=self.size,
-                                         frame=frame,
-                                         ri=ri,
-                                         self=self, result=result)
-            result.setitem(ri.offset, sig.eval(frame, self))
-            frame.next(shapelen)
-            ri = ri.next(shapelen)
-        return result
+        ra = ResultArray(self, self.size, self.shape, self.res_dtype)
+        loop.compute(ra)
+        return ra.left
 
     def force_if_needed(self):
         if self.forced_result is None:
     def create_sig(self):
         if self.forced_result is not None:
             return self.forced_result.create_sig()
-        return signature.Call1(self.ufunc, self.name, self.values.create_sig())
+        return signature.Call1(self.ufunc, self.name, self.calc_dtype,
+                               self.values.create_sig())
 
 class Call2(VirtualArray):
     """
         return signature.Call2(self.ufunc, self.name, self.calc_dtype,
                                self.left.create_sig(), self.right.create_sig())
 
+class ResultArray(Call2):
+    def __init__(self, child, size, shape, dtype, res=None, order='C'):
+        if res is None:
+            res = W_NDimArray(size, shape, dtype, order)
+        Call2.__init__(self, None, 'assign', shape, dtype, dtype, res, child)
+
+    def create_sig(self):
+        return signature.ResultSignature(self.res_dtype, self.left.create_sig(),
+                                         self.right.create_sig())
+
+def done_if_true(dtype, val):
+    return dtype.itemtype.bool(val)
+
+def done_if_false(dtype, val):
+    return not dtype.itemtype.bool(val)
+
+class ReduceArray(Call2):
+    def __init__(self, func, name, identity, child, dtype):
+        self.identity = identity
+        Call2.__init__(self, func, name, [1], dtype, dtype, None, child)
+
+    def compute_first_step(self, sig, frame):
+        assert isinstance(sig, signature.ReduceSignature)
+        if self.identity is None:
+            frame.cur_value = sig.right.eval(frame, self.right).convert_to(
+                self.calc_dtype)
+            frame.next(len(self.right.shape))
+        else:
+            frame.cur_value = self.identity.convert_to(self.calc_dtype)
+
+    def create_sig(self):
+        if self.name == 'logical_and':
+            done_func = done_if_false
+        elif self.name == 'logical_or':
+            done_func = done_if_true
+        else:
+            done_func = None
+        return signature.ReduceSignature(self.ufunc, self.name, self.res_dtype,
+                                 signature.ScalarSignature(self.res_dtype),
+                                         self.right.create_sig(), done_func)
+
+class AxisReduce(Call2):
+    _immutable_fields_ = ['left', 'right']
+
+    def __init__(self, ufunc, name, identity, shape, dtype, left, right, dim):
+        Call2.__init__(self, ufunc, name, shape, dtype, dtype,
+                       left, right)
+        self.dim = dim
+        self.identity = identity
+
+    def compute_first_step(self, sig, frame):
+        if self.identity is not None:
+            frame.identity = self.identity.convert_to(self.calc_dtype)
+
+    def create_sig(self):
+        return signature.AxisReduceSignature(self.ufunc, self.name,
+                                             self.res_dtype,
+                                 signature.ScalarSignature(self.res_dtype),
+                                             self.right.create_sig())
+
 class SliceArray(Call2):
     def __init__(self, shape, dtype, left, right, no_broadcast=False):
         self.no_broadcast = no_broadcast
                                             self.calc_dtype,
                                             lsig, rsig)
 
-class AxisReduce(Call2):
-    """ NOTE: this is only used as a container, you should never
-    encounter such things in the wild. Remove this comment
-    when we'll make AxisReduce lazy
-    """
-    _immutable_fields_ = ['left', 'right']
-
-    def __init__(self, ufunc, name, shape, dtype, left, right, dim):
-        Call2.__init__(self, ufunc, name, shape, dtype, dtype,
-                       left, right)
-        self.dim = dim
-
 class ConcreteArray(BaseArray):
     """ An array that have actual storage, whether owned or not
     """
             self._fast_setslice(space, w_value)
         else:
             arr = SliceArray(self.shape, self.dtype, self, w_value)
-            self._sliceloop(arr)
+            loop.compute(arr)
 
     def _fast_setslice(self, space, w_value):
         assert isinstance(w_value, ConcreteArray)
                 source.next()
                 dest.next()
 
-    def _sliceloop(self, arr):
-        sig = arr.find_sig()
-        frame = sig.create_frame(arr)
-        shapelen = len(self.shape)
-        while not frame.done():
-            slice_driver.jit_merge_point(sig=sig, frame=frame, self=self,
-                                         arr=arr,
-                                         shapelen=shapelen)
-            sig.eval(frame, arr)
-            frame.next(shapelen)
-
     def copy(self, space):
         array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
         array.setslice(space, self)
                            parent.order, parent)
         self.start = start
 
-    def create_iter(self):
+    def create_iter(self, transforms=None):
         return ViewIterator(self.start, self.strides, self.backstrides,
-                            self.shape)
+                            self.shape).apply_transformations(self, transforms)
 
     def setshape(self, space, new_shape):
         if len(self.shape) < 1:
         self.shape = new_shape
         self.calc_strides(new_shape)
 
-    def create_iter(self):
-        return ArrayIterator(self.size)
+    def create_iter(self, transforms=None):
+        return ArrayIterator(self.size).apply_transformations(self, transforms)
 
     def create_sig(self):
         return signature.ArraySignature(self.dtype)
                            BaseArray.descr_set_shape),
     size = GetSetProperty(BaseArray.descr_get_size),
     ndim = GetSetProperty(BaseArray.descr_get_ndim),
-    item = interp2app(BaseArray.descr_item),
+    itemsize = GetSetProperty(BaseArray.descr_get_itemsize),
+    nbytes = GetSetProperty(BaseArray.descr_get_nbytes),
 
     T = GetSetProperty(BaseArray.descr_get_transpose),
     flat = GetSetProperty(BaseArray.descr_get_flatiter),
     ravel = interp2app(BaseArray.descr_ravel),
+    item = interp2app(BaseArray.descr_item),
 
     mean = interp2app(BaseArray.descr_mean),
     sum = interp2app(BaseArray.descr_sum),
     def descr_iter(self):
         return self
 
+    def descr_len(self, space):
+        return space.wrap(self.size)
+
     def descr_index(self, space):
         return space.wrap(self.index)
 
     def descr_coords(self, space):
-        coords, step, lngth = to_coords(space, self.base.shape, 
-                            self.base.size, self.base.order, 
+        coords, step, lngth = to_coords(space, self.base.shape,
+                            self.base.size, self.base.order,
                             space.wrap(self.index))
         return space.newtuple([space.wrap(c) for c in coords])
 
                                              step=step,
                                              res=res,
                                              ri=ri,
-                                            ) 
+                                            )
             w_val = base.getitem(basei.offset)
             res.setitem(ri.offset,w_val)
             basei = basei.next_skip_x(shapelen, step)
                                              arr=arr,
                                              ai=ai,
                                              lngth=lngth,
-                                            ) 
+                                            )
             v = arr.getitem(ai).convert_to(base.dtype)
             base.setitem(basei.offset, v)
             # need to repeat input values until all assignments are done
     def create_sig(self):
         return signature.FlatSignature(self.base.dtype)
 
+    def create_iter(self, transforms=None):
+        return ViewIterator(self.base.start, self.base.strides,
+                    self.base.backstrides,
+                    self.base.shape).apply_transformations(self.base,
+                                                           transforms)
+
     def descr_base(self, space):
         return space.wrap(self.base)
 
 W_FlatIterator.typedef = TypeDef(
     'flatiter',
-    #__array__  = #MISSING
     __iter__ = interp2app(W_FlatIterator.descr_iter),
+    __len__ = interp2app(W_FlatIterator.descr_len),
     __getitem__ = interp2app(W_FlatIterator.descr_getitem),
     __setitem__ = interp2app(W_FlatIterator.descr_setitem),
+
     __eq__ = interp2app(BaseArray.descr_eq),
     __ne__ = interp2app(BaseArray.descr_ne),
     __lt__ = interp2app(BaseArray.descr_lt),
     __le__ = interp2app(BaseArray.descr_le),
     __gt__ = interp2app(BaseArray.descr_gt),
     __ge__ = interp2app(BaseArray.descr_ge),
-    #__sizeof__ #MISSING
+
     base = GetSetProperty(W_FlatIterator.descr_base),
     index = GetSetProperty(W_FlatIterator.descr_index),
     coords = GetSetProperty(W_FlatIterator.descr_coords),

pypy/module/micronumpy/interp_ufuncs.py

 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
-from pypy.module.micronumpy import interp_boxes, interp_dtype, support
-from pypy.module.micronumpy.signature import (ReduceSignature, find_sig,
-    new_printable_location, AxisReduceSignature, ScalarSignature)
-from pypy.rlib import jit
+from pypy.module.micronumpy import interp_boxes, interp_dtype, support, loop
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
-
-reduce_driver = jit.JitDriver(
-    greens=['shapelen', "sig"],
-    virtualizables=["frame"],
-    reds=["frame", "self", "dtype", "value", "obj"],
-    get_printable_location=new_printable_location('reduce'),
-    name='numpy_reduce',
-)
-
-axisreduce_driver = jit.JitDriver(
-    greens=['shapelen', 'sig'],
-    virtualizables=['frame'],
-    reds=['self','arr', 'identity', 'frame'],
-    name='numpy_axisreduce',
-    get_printable_location=new_printable_location('axisreduce'),
-)
-
-
 class W_Ufunc(Wrappable):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
     def reduce(self, space, w_obj, multidim, promote_to_largest, dim,
                keepdims=False):
         from pypy.module.micronumpy.interp_numarray import convert_to_array, \
-                                                           Scalar
+                                                           Scalar, ReduceArray
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
         if isinstance(obj, Scalar):
             raise OperationError(space.w_TypeError, space.wrap("cannot reduce "
                 "on a scalar"))
-
         size = obj.size
-        dtype = find_unaryop_result_dtype(
-            space, obj.find_dtype(),
-            promote_to_float=self.promote_to_float,
-            promote_to_largest=promote_to_largest,
-            promote_bools=True
-        )
+        if self.comparison_func:
+            dtype = interp_dtype.get_dtype_cache(space).w_booldtype
+        else:
+            dtype = find_unaryop_result_dtype(
+                space, obj.find_dtype(),
+                promote_to_float=self.promote_to_float,
+                promote_to_largest=promote_to_largest,
+                promote_bools=True
+            )
         shapelen = len(obj.shape)
         if self.identity is None and size == 0:
             raise operationerrfmt(space.w_ValueError, "zero-size array to "
                     "%s.reduce without identity", self.name)
         if shapelen > 1 and dim >= 0:
-            res = self.do_axis_reduce(obj, dtype, dim, keepdims)
-            return space.wrap(res)
-        scalarsig = ScalarSignature(dtype)
-        sig = find_sig(ReduceSignature(self.func, self.name, dtype,
-                                       scalarsig,
-                                       obj.create_sig()), obj)
-        frame = sig.create_frame(obj)
-        if self.identity is None:
-            value = sig.eval(frame, obj).convert_to(dtype)
-            frame.next(shapelen)
-        else:
-            value = self.identity.convert_to(dtype)
-        return self.reduce_loop(shapelen, sig, frame, value, obj, dtype)
+            return self.do_axis_reduce(obj, dtype, dim, keepdims)
+        arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
+        return loop.compute(arr)
 
     def do_axis_reduce(self, obj, dtype, dim, keepdims):
         from pypy.module.micronumpy.interp_numarray import AxisReduce,\
              W_NDimArray
-
         if keepdims:
             shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:]
         else:
             shape = obj.shape[:dim] + obj.shape[dim + 1:]
         result = W_NDimArray(support.product(shape), shape, dtype)
-        rightsig = obj.create_sig()
-        # note - this is just a wrapper so signature can fetch
-        #        both left and right, nothing more, especially
-        #        this is not a true virtual array, because shapes
-        #        don't quite match
-        arr = AxisReduce(self.func, self.name, obj.shape, dtype,
+        arr = AxisReduce(self.func, self.name, self.identity, obj.shape, dtype,
                          result, obj, dim)
-        scalarsig = ScalarSignature(dtype)
-        sig = find_sig(AxisReduceSignature(self.func, self.name, dtype,
-                                           scalarsig, rightsig), arr)
-        assert isinstance(sig, AxisReduceSignature)
-        frame = sig.create_frame(arr)
-        shapelen = len(obj.shape)
-        if self.identity is not None:
-            identity = self.identity.convert_to(dtype)
-        else:
-            identity = None
-        self.reduce_axis_loop(frame, sig, shapelen, arr, identity)
-        return result
-
-    def reduce_axis_loop(self, frame, sig, shapelen, arr, identity):
-        # note - we can be advanterous here, depending on the exact field
-        # layout. For now let's say we iterate the original way and
-        # simply follow the original iteration order
-        while not frame.done():
-            axisreduce_driver.jit_merge_point(frame=frame, self=self,
-                                              sig=sig,
-                                              identity=identity,
-                                              shapelen=shapelen, arr=arr)
-            iterator = frame.get_final_iter()
-            v = sig.eval(frame, arr).convert_to(sig.calc_dtype)
-            if iterator.first_line:
-                if identity is not None:
-                    value = self.func(sig.calc_dtype, identity, v)
-                else:
-                    value = v
-            else:
-                cur = arr.left.getitem(iterator.offset)
-                value = self.func(sig.calc_dtype, cur, v)
-            arr.left.setitem(iterator.offset, value)
-            frame.next(shapelen)
-
-    def reduce_loop(self, shapelen, sig, frame, value, obj, dtype):
-        while not frame.done():