Commits

Armin Rigo committed b702950 Merge

hg merge default

  • Participants
  • Parent commits f07d5e9, 2e5b6dc
  • Branches SpecialisedTuples

Comments (0)

Files changed (36)

lib_pypy/distributed/socklayer.py

 import py
 from socket import socket
 
-XXX needs import adaptation as 'green' is removed from py lib for years 
+raise ImportError("XXX needs import adaptation as 'green' is removed from py lib for years")
 from py.impl.green.msgstruct import decodemessage, message
 from socket import socket, AF_INET, SOCK_STREAM
 import marshal
     def setup(self):
         super(AppClassCollector, self).setup()
         cls = self.obj
+        #
+        # <hack>
+        for name in dir(cls):
+            if name.startswith('test_'):
+                func = getattr(cls, name, None)
+                code = getattr(func, 'func_code', None)
+                if code and code.co_flags & 32:
+                    raise AssertionError("unsupported: %r is a generator "
+                                         "app-level test method" % (name,))
+        # </hack>
+        #
         space = cls.space
         clsname = cls.__name__
         if self.config.option.runappdirect:
 # built documents.
 #
 # The short X.Y version.
-version = '1.6'
+version = '1.7'
 # The full version, including alpha/beta/rc tags.
-release = '1.6'
+release = '1.7'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pypy/interpreter/baseobjspace.py

         'parser', 'fcntl', '_codecs', 'binascii'
     ]
 
+    # These modules are treated like CPython treats built-in modules,
+    # i.e. they always shadow any xx.py.  The other modules are treated
+    # like CPython treats extension modules, and are loaded in sys.path
+    # order by the fake entry '.../lib_pypy/__extensions__'.
+    MODULES_THAT_ALWAYS_SHADOW = dict.fromkeys([
+        '__builtin__', '__pypy__', '_ast', '_codecs', '_sre', '_warnings',
+        '_weakref', 'errno', 'exceptions', 'gc', 'imp', 'marshal',
+        'posix', 'nt', 'pwd', 'signal', 'sys', 'thread', 'zipimport',
+    ], None)
+
     def make_builtins(self):
         "NOT_RPYTHON: only for initializing the space."
 

pypy/jit/backend/llsupport/regalloc.py

     """ Manage frame positions
     """
     def __init__(self):
-        self.frame_bindings = {}
-        self.frame_depth    = 0
+        self.bindings = {}
+        self.used = []      # list of bools
+        self.hint_frame_locations = {}
+
+    frame_depth = property(lambda:xxx, lambda:xxx)   # XXX kill me
+
+    def get_frame_depth(self):
+        return len(self.used)
 
     def get(self, box):
-        return self.frame_bindings.get(box, None)
+        return self.bindings.get(box, None)
 
     def loc(self, box):
+        """Return or create the frame location associated with 'box'."""
+        # first check if it's already in the frame_manager
         try:
-            return self.frame_bindings[box]
+            return self.bindings[box]
         except KeyError:
-            return self.get_new_loc(box)
+            pass
+        # check if we have a hint for this box
+        if box in self.hint_frame_locations:
+            # if we do, try to reuse the location for this box
+            loc = self.hint_frame_locations[box]
+            if self.try_to_reuse_location(box, loc):
+                return loc
+        # no valid hint.  make up a new free location
+        return self.get_new_loc(box)
 
     def get_new_loc(self, box):
         size = self.frame_size(box.type)
-        self.frame_depth += ((-self.frame_depth) & (size-1))
-        # ^^^ frame_depth is rounded up to a multiple of 'size', assuming
+        # frame_depth is rounded up to a multiple of 'size', assuming
         # that 'size' is a power of two.  The reason for doing so is to
         # avoid obscure issues in jump.py with stack locations that try
         # to move from position (6,7) to position (7,8).
-        newloc = self.frame_pos(self.frame_depth, box.type)
-        self.frame_bindings[box] = newloc
-        self.frame_depth += size
+        while self.get_frame_depth() & (size - 1):
+            self.used.append(False)
+        #
+        index = self.get_frame_depth()
+        newloc = self.frame_pos(index, box.type)
+        for i in range(size):
+            self.used.append(True)
+        #
+        if not we_are_translated():    # extra testing
+            testindex = self.get_loc_index(newloc)
+            assert testindex == index
+        #
+        self.bindings[box] = newloc
         return newloc
 
+    def set_binding(self, box, loc):
+        self.bindings[box] = loc
+        #
+        index = self.get_loc_index(loc)
+        endindex = index + self.frame_size(box.type)
+        while len(self.used) < endindex:
+            self.used.append(False)
+        while index < endindex:
+            self.used[index] = True
+            index += 1
+
     def reserve_location_in_frame(self, size):
-        frame_depth = self.frame_depth
-        self.frame_depth += size
+        frame_depth = self.get_frame_depth()
+        for i in range(size):
+            self.used.append(True)
         return frame_depth
 
+    def mark_as_free(self, box):
+        try:
+            loc = self.bindings[box]
+        except KeyError:
+            return    # already gone
+        del self.bindings[box]
+        #
+        size = self.frame_size(box.type)
+        baseindex = self.get_loc_index(loc)
+        for i in range(size):
+            index = baseindex + i
+            assert 0 <= index < len(self.used)
+            self.used[index] = False
+
+    def try_to_reuse_location(self, box, loc):
+        index = self.get_loc_index(loc)
+        assert index >= 0
+        size = self.frame_size(box.type)
+        for i in range(size):
+            while (index + i) >= len(self.used):
+                self.used.append(False)
+            if self.used[index + i]:
+                return False    # already in use
+        # good, we can reuse the location
+        for i in range(size):
+            self.used[index + i] = True
+        self.bindings[box] = loc
+        return True
+
     # abstract methods that need to be overwritten for specific assemblers
     @staticmethod
     def frame_pos(loc, type):
     @staticmethod
     def frame_size(type):
         return 1
+    @staticmethod
+    def get_loc_index(loc):
+        raise NotImplementedError("Purely abstract")
+
 
 class RegisterManager(object):
     """ Class that keeps track of register allocations
         self.position = -1
         self.frame_manager = frame_manager
         self.assembler = assembler
-        self.hint_frame_locations = {}     # {Box: StackLoc}
-        self.freed_frame_locations = {}    # {StackLoc: None}
 
     def is_still_alive(self, v):
         # Check if 'v' is alive at the current position.
                 self.free_regs.append(self.reg_bindings[v])
                 del self.reg_bindings[v]
             if self.frame_manager is not None:
-                if v in self.frame_manager.frame_bindings:
-                    loc = self.frame_manager.frame_bindings[v]
-                    self.freed_frame_locations[loc] = None
+                self.frame_manager.mark_as_free(v)
 
     def possibly_free_vars(self, vars):
         """ Same as 'possibly_free_var', but for all v in vars.
                 self.reg_bindings[v] = loc
                 return loc
 
-    def _frame_loc(self, v):
-        # first check if it's already in the frame_manager
-        try:
-            return self.frame_manager.frame_bindings[v]
-        except KeyError:
-            pass
-        # check if we have a hint for this box
-        if v in self.hint_frame_locations:
-            # if we do, check that the hinted location is known to be free
-            loc = self.hint_frame_locations[v]
-            if loc in self.freed_frame_locations:
-                del self.freed_frame_locations[loc]
-                self.frame_manager.frame_bindings[v] = loc
-                return loc
-        # no valid hint.  make up a new free location
-        return self.frame_manager.get_new_loc(v)
-
     def _spill_var(self, v, forbidden_vars, selected_reg,
                    need_lower_byte=False):
         v_to_spill = self._pick_variable_to_spill(v, forbidden_vars,
         loc = self.reg_bindings[v_to_spill]
         del self.reg_bindings[v_to_spill]
         if self.frame_manager.get(v_to_spill) is None:
-            newloc = self._frame_loc(v_to_spill)
+            newloc = self.frame_manager.loc(v_to_spill)
             self.assembler.regalloc_mov(loc, newloc)
         return loc
 
         except KeyError:
             if box in self.bindings_to_frame_reg:
                 return self.frame_reg
-            return self._frame_loc(box)
+            return self.frame_manager.loc(box)
 
     def return_constant(self, v, forbidden_vars=[], selected_reg=None):
         """ Return the location of the constant v.  If 'selected_reg' is
             self.reg_bindings[v] = loc
             self.assembler.regalloc_mov(prev_loc, loc)
         else:
-            loc = self._frame_loc(v)
+            loc = self.frame_manager.loc(v)
             self.assembler.regalloc_mov(prev_loc, loc)
 
     def force_result_in_reg(self, result_v, v, forbidden_vars=[]):
             self.reg_bindings[result_v] = loc
             return loc
         if v not in self.reg_bindings:
-            prev_loc = self._frame_loc(v)
+            prev_loc = self.frame_manager.loc(v)
             loc = self.force_allocate_reg(v, forbidden_vars)
             self.assembler.regalloc_mov(prev_loc, loc)
         assert v in self.reg_bindings
     def _sync_var(self, v):
         if not self.frame_manager.get(v):
             reg = self.reg_bindings[v]
-            to = self._frame_loc(v)
+            to = self.frame_manager.loc(v)
             self.assembler.regalloc_mov(reg, to)
         # otherwise it's clean
 

pypy/jit/backend/llsupport/test/test_regalloc.py

     def frame_size(self, box_type):
         if box_type == FLOAT:
             return 2
+        elif box_type == INT:
+            return 1
         else:
-            return 1
+            raise ValueError(box_type)
+    def get_loc_index(self, loc):
+        assert isinstance(loc, FakeFramePos)
+        return loc.pos
 
 class MockAsm(object):
     def __init__(self):
             rm.force_allocate_reg(b)
         rm.before_call()
         assert len(rm.reg_bindings) == 2
-        assert fm.frame_depth == 2
+        assert fm.get_frame_depth() == 2
         assert len(asm.moves) == 2
         rm._check_invariants()
         rm.after_call(boxes[-1])
             rm.force_allocate_reg(b)
         rm.before_call(save_all_regs=True)
         assert len(rm.reg_bindings) == 0
-        assert fm.frame_depth == 4
+        assert fm.get_frame_depth() == 4
         assert len(asm.moves) == 4
         rm._check_invariants()
         rm.after_call(boxes[-1])
         xrm = XRegisterManager(longevity, frame_manager=fm, assembler=asm)
         xrm.loc(f0)
         rm.loc(b0)
-        assert fm.frame_depth == 3
+        assert fm.get_frame_depth() == 3
         
         
 
 
 
     def test_hint_frame_locations_1(self):
-        b0, b1 = newboxes(0, 1)
-        longevity = {b0: (0, 1), b1: (0, 1)}
+        b0, = newboxes(0)
         fm = TFrameManager()
-        asm = MockAsm()
-        rm = RegisterManager(longevity, frame_manager=fm, assembler=asm)
-        rm.hint_frame_locations[b0] = "some_stack_loc"
-        rm.freed_frame_locations["some_stack_loc"] = None
-        rm.force_allocate_reg(b0)
-        rm.force_allocate_reg(b1)
-        rm.force_spill_var(b0)
-        rm.force_spill_var(b1)
-        assert rm.loc(b0) == "some_stack_loc"
-        assert isinstance(rm.loc(b1), FakeFramePos)
-        rm._check_invariants()
+        loc123 = FakeFramePos(123, INT)
+        fm.hint_frame_locations[b0] = loc123
+        assert fm.get_frame_depth() == 0
+        loc = fm.loc(b0)
+        assert loc == loc123
+        assert fm.get_frame_depth() == 124
 
     def test_hint_frame_locations_2(self):
         b0, b1, b2 = newboxes(0, 1, 2)
         rm.force_spill_var(b0)
         loc = rm.loc(b0)
         assert isinstance(loc, FakeFramePos)
+        assert fm.get_loc_index(loc) == 0
         rm.position = 1
-        assert loc not in rm.freed_frame_locations
+        assert fm.used == [True]
         rm.possibly_free_var(b0)
-        assert loc in rm.freed_frame_locations
+        assert fm.used == [False]
         #
-        rm.hint_frame_locations[b1] = loc
+        fm.hint_frame_locations[b1] = loc
         rm.force_spill_var(b1)
         loc1 = rm.loc(b1)
-        assert loc1 is loc
-        assert rm.freed_frame_locations == {}
+        assert loc1 == loc
+        assert fm.used == [True]
         #
-        rm.hint_frame_locations[b2] = loc
+        fm.hint_frame_locations[b2] = loc
         rm.force_spill_var(b2)
         loc2 = rm.loc(b2)
-        assert loc2 is not loc1     # because it's not in freed_frame_locations
+        assert loc2 != loc1     # because it was not free
+        assert fm.used == [True, True]
         #
         rm._check_invariants()
+
+    def test_frame_manager_basic(self):
+        b0, b1 = newboxes(0, 1)
+        fm = TFrameManager()
+        loc0 = fm.loc(b0)
+        assert fm.get_loc_index(loc0) == 0
+        #
+        assert fm.get(b1) is None
+        loc1 = fm.loc(b1)
+        assert fm.get_loc_index(loc1) == 1
+        assert fm.get(b1) == loc1
+        #
+        loc0b = fm.loc(b0)
+        assert loc0b == loc0
+        #
+        fm.loc(BoxInt())
+        assert fm.get_frame_depth() == 3
+        #
+        f0 = BoxFloat()
+        locf0 = fm.loc(f0)
+        assert fm.get_loc_index(locf0) == 4
+        assert fm.get_frame_depth() == 6
+        #
+        f1 = BoxFloat()
+        locf1 = fm.loc(f1)
+        assert fm.get_loc_index(locf1) == 6
+        assert fm.get_frame_depth() == 8
+        assert fm.used == [True, True, True, False, True, True, True, True]
+        #
+        fm.mark_as_free(b0)
+        assert fm.used == [False, True, True, False, True, True, True, True]
+        fm.mark_as_free(b0)
+        assert fm.used == [False, True, True, False, True, True, True, True]
+        fm.mark_as_free(f1)
+        assert fm.used == [False, True, True, False, True, True, False, False]
+        #
+        fm.reserve_location_in_frame(1)
+        assert fm.get_frame_depth() == 9
+        assert fm.used == [False, True, True, False, True, True, False, False, True]
+        #
+        assert b0 not in fm.bindings
+        fm.set_binding(b0, loc0)
+        assert b0 in fm.bindings
+        assert fm.used == [True, True, True, False, True, True, False, False, True]
+        #
+        b3 = BoxInt()
+        assert not fm.try_to_reuse_location(b3, loc0)
+        assert fm.used == [True, True, True, False, True, True, False, False, True]
+        #
+        fm.mark_as_free(b0)
+        assert fm.used == [False, True, True, False, True, True, False, False, True]
+        assert fm.try_to_reuse_location(b3, loc0)
+        assert fm.used == [True, True, True, False, True, True, False, False, True]
+        #
+        fm.mark_as_free(b0)   # already free
+        assert fm.used == [True, True, True, False, True, True, False, False, True]
+        #
+        fm.mark_as_free(b3)
+        assert fm.used == [False, True, True, False, True, True, False, False, True]
+        f3 = BoxFloat()
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(0, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(1, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(2, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(3, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(4, FLOAT))
+        assert not fm.try_to_reuse_location(f3, fm.frame_pos(5, FLOAT))
+        assert fm.used == [False, True, True, False, True, True, False, False, True]
+        assert fm.try_to_reuse_location(f3, fm.frame_pos(6, FLOAT))
+        assert fm.used == [False, True, True, False, True, True, True, True, True]
+        #
+        fm.used = [False]
+        assert fm.try_to_reuse_location(BoxFloat(), fm.frame_pos(0, FLOAT))
+        assert fm.used == [True, True]
+        #
+        fm.used = [True]
+        assert not fm.try_to_reuse_location(BoxFloat(), fm.frame_pos(0, FLOAT))
+        assert fm.used == [True]

pypy/jit/backend/x86/assembler.py

         regalloc.walk_operations(operations)
         if we_are_translated() or self.cpu.dont_keepalive_stuff:
             self._regalloc = None   # else keep it around for debugging
-        frame_depth = regalloc.fm.frame_depth
+        frame_depth = regalloc.fm.get_frame_depth()
         param_depth = regalloc.param_depth
         jump_target_descr = regalloc.jump_target_descr
         if jump_target_descr is not None:

pypy/jit/backend/x86/regalloc.py

             return 2
         else:
             return 1
+    @staticmethod
+    def get_loc_index(loc):
+        assert isinstance(loc, StackLoc)
+        return loc.position
 
 if WORD == 4:
     gpr_reg_mgr_cls = X86RegisterManager
                        allgcrefs):
         operations, _ = self._prepare(inputargs, operations, allgcrefs)
         self._update_bindings(arglocs, inputargs)
-        self.fm.frame_depth = prev_depths[0]
         self.param_depth = prev_depths[1]
         return operations
 
                     self.xrm.reg_bindings[arg] = loc
                     used[loc] = None
                 else:
-                    self.fm.frame_bindings[arg] = loc
+                    self.fm.set_binding(arg, loc)
             else:
                 if isinstance(loc, RegLoc):
                     if loc is ebp:
                         self.rm.reg_bindings[arg] = loc
                         used[loc] = None
                 else:
-                    self.fm.frame_bindings[arg] = loc
+                    self.fm.set_binding(arg, loc)
         self.rm.free_regs = []
         for reg in self.rm.all_regs:
             if reg not in used:
     def get_current_depth(self):
         # return (self.fm.frame_depth, self.param_depth), but trying to share
         # the resulting tuple among several calls
-        arg0 = self.fm.frame_depth
+        arg0 = self.fm.get_frame_depth()
         arg1 = self.param_depth
         result = self.assembler._current_depths_cache
         if result[0] != arg0 or result[1] != arg1:
                 loc = nonfloatlocs[i]
                 if isinstance(loc, StackLoc):
                     assert box.type != FLOAT
-                    self.rm.hint_frame_locations[box] = loc
+                    self.fm.hint_frame_locations[box] = loc
                 else:
                     loc = floatlocs[i]
                     if isinstance(loc, StackLoc):
                         assert box.type == FLOAT
-                        self.xrm.hint_frame_locations[box] = loc
+                        self.fm.hint_frame_locations[box] = loc
 
     def consider_jump(self, op):
         assembler = self.assembler
 
     def get_mark_gc_roots(self, gcrootmap, use_copy_area=False):
         shape = gcrootmap.get_basic_shape(IS_X86_64)
-        for v, val in self.fm.frame_bindings.items():
+        for v, val in self.fm.bindings.items():
             if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)):
                 assert isinstance(val, StackLoc)
                 gcrootmap.add_frame_offset(shape, get_ebp_ofs(val.position))

pypy/jit/backend/x86/test/test_recompilation.py

         i5 = int_add(i4, 1)
         i6 = int_add(i5, 1)
         i7 = int_add(i5, i4)
+        force_spill(i5)
         i8 = int_add(i7, 1)
         i9 = int_add(i8, 1)
         finish(i3, i4, i5, i6, i7, i8, i9, descr=fdescr2)
         bridge = self.attach_bridge(ops, loop, -2)
         descr = loop.operations[2].getdescr()
         new = descr._x86_bridge_frame_depth
-        assert descr._x86_bridge_param_depth == 0        
-        # XXX: Maybe add enough ops to force stack on 64-bit as well?
-        if IS_X86_32:
-            assert new > previous
+        assert descr._x86_bridge_param_depth == 0
+        # the force_spill() forces the stack to grow
+        assert new > previous
         self.cpu.set_future_value_int(0, 0)
         fail = self.run(loop)
         assert fail.identifier == 2
         i8 = int_add(i3, 1)
         i6 = int_add(i8, i10)
         i7 = int_add(i3, i6)
+        force_spill(i6)
+        force_spill(i7)
+        force_spill(i8)
         i12 = int_add(i7, i8)
         i11 = int_add(i12, i6)
         jump(i3, i12, i11, i10, i6, i7, descr=looptoken)
         guard_op = loop.operations[5]
         loop_frame_depth = loop.token._x86_frame_depth
         assert loop.token._x86_param_depth == 0
-        # XXX: Maybe add enough ops to force stack on 64-bit as well?
-        if IS_X86_32:
-            assert guard_op.getdescr()._x86_bridge_frame_depth > loop_frame_depth
+        # the force_spill() forces the stack to grow
+        assert guard_op.getdescr()._x86_bridge_frame_depth > loop_frame_depth
         assert guard_op.getdescr()._x86_bridge_param_depth == 0
         self.cpu.set_future_value_int(0, 0)
         self.cpu.set_future_value_int(1, 0)

pypy/jit/metainterp/test/test_del.py

 import py
-from pypy.rlib.jit import JitDriver
+from pypy.rlib.jit import JitDriver, dont_look_inside
+from pypy.rlib.objectmodel import keepalive_until_here
+from pypy.rlib import rgc
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 
 
         assert res == 1
         self.check_resops(call=1)   # for the case B(), but not for the case A()
 
+    def test_keepalive(self):
+        py.test.skip("XXX fails")   # hum, I think the test itself is broken
+        #
+        mydriver = JitDriver(reds = ['n', 'states'], greens = [])
+        class State:
+            num = 1
+        class X:
+            def __init__(self, state):
+                self.state = state
+            def __del__(self):
+                self.state.num += 1
+        @dont_look_inside
+        def do_stuff():
+            pass
+        def f(n):
+            states = []
+            while n > 0:
+                mydriver.jit_merge_point(n=n, states=states)
+                state = State()
+                states.append(state)
+                x = X(state)
+                do_stuff()
+                state.num *= 1000
+                do_stuff()
+                keepalive_until_here(x)
+                n -= 1
+            return states
+        def main(n):
+            states = f(n)
+            rgc.collect()
+            rgc.collect()
+            err = 1001
+            for state in states:
+                if state.num != 1001:
+                    err = state.num
+                    print 'ERROR:', err
+            return err
+        assert main(20) == 1001
+        res = self.meta_interp(main, [20])
+        assert res == 1001
+
 
 class TestLLtype(DelTests, LLJitMixin):
     def test_signal_action(self):

pypy/jit/metainterp/test/test_fficall.py

+from __future__ import with_statement
 import py
 
 from pypy.jit.metainterp.test.support import LLJitMixin

pypy/module/_hashlib/interp_hashlib.py

 
 class W_Hash(Wrappable):
     ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
-    _block_size = -1
 
     def __init__(self, space, name):
         self.name = name
-        self.digest_size = self.compute_digest_size()
+        digest_type = self.digest_type_by_name(space)
+        self.digest_size = rffi.getintfield(digest_type, 'c_md_size')
 
         # Allocate a lock for each HASH object.
         # An optimization would be to not release the GIL on small requests,
 
         ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
         rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size)
+        ropenssl.EVP_DigestInit(ctx, digest_type)
         self.ctx = ctx
 
-    def initdigest(self, space, name):
-        digest = ropenssl.EVP_get_digestbyname(name)
-        if not digest:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("unknown hash function"))
-        ropenssl.EVP_DigestInit(self.ctx, digest)
-
     def __del__(self):
         # self.lock.free()
         if self.ctx:
             ropenssl.EVP_MD_CTX_cleanup(self.ctx)
             lltype.free(self.ctx, flavor='raw')
 
+    def digest_type_by_name(self, space):
+        digest_type = ropenssl.EVP_get_digestbyname(self.name)
+        if not digest_type:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("unknown hash function"))
+        return digest_type
+
     def descr_repr(self, space):
         addrstring = self.getaddrstring(space)
         return space.wrap("<%s HASH object at 0x%s>" % (
         return space.wrap(self.digest_size)
 
     def get_block_size(self, space):
-        return space.wrap(self.compute_block_size())
+        digest_type = self.digest_type_by_name(space)
+        block_size = rffi.getintfield(digest_type, 'c_block_size')
+        return space.wrap(block_size)
 
     def _digest(self, space):
         with lltype.scoped_alloc(ropenssl.EVP_MD_CTX.TO) as ctx:
                 ropenssl.EVP_MD_CTX_cleanup(ctx)
                 return rffi.charpsize2str(digest, digest_size)
 
-    def compute_digest_size(self):
-        # XXX This isn't the nicest way, but the EVP_MD_size OpenSSL
-        # XXX function is defined as a C macro on OS X and would be
-        # XXX significantly harder to implement in another way.
-        # Values are digest sizes in bytes
-        return {
-            'md5':    16, 'MD5':    16,
-            'sha1':   20, 'SHA1':   20,
-            'sha224': 28, 'SHA224': 28,
-            'sha256': 32, 'SHA256': 32,
-            'sha384': 48, 'SHA384': 48,
-            'sha512': 64, 'SHA512': 64,
-            }.get(self.name, 0)
-
-    def compute_block_size(self):
-        if self._block_size != -1:
-            return self._block_size
-        # XXX This isn't the nicest way, but the EVP_MD_CTX_block_size
-        # XXX OpenSSL function is defined as a C macro on some systems
-        # XXX and would be significantly harder to implement in
-        # XXX another way.
-        self._block_size = {
-            'md5':     64, 'MD5':     64,
-            'sha1':    64, 'SHA1':    64,
-            'sha224':  64, 'SHA224':  64,
-            'sha256':  64, 'SHA256':  64,
-            'sha384': 128, 'SHA384': 128,
-            'sha512': 128, 'SHA512': 128,
-            }.get(self.name, 0)
-        return self._block_size
 
 W_Hash.typedef = TypeDef(
     'HASH',
     digestsize=GetSetProperty(W_Hash.get_digest_size),
     block_size=GetSetProperty(W_Hash.get_block_size),
     )
+W_Hash.acceptable_as_base_class = False
 
 @unwrap_spec(name=str, string='bufferstr')
 def new(space, name, string=''):
     w_hash = W_Hash(space, name)
-    w_hash.initdigest(space, name)
     w_hash.update(space, string)
     return space.wrap(w_hash)
 
         return new(space, name, string)
     return new_hash
 
-for name in algorithms:
-    newname = 'new_%s' % (name,)
-    globals()[newname] = make_new_hash(name, newname)
+for _name in algorithms:
+    _newname = 'new_%s' % (_name,)
+    globals()[_newname] = make_new_hash(_name, _newname)

pypy/module/_hashlib/test/test_hashlib.py

         assert h.digest() == _hashlib.openssl_md5('x' * 20).digest()
         _hashlib.openssl_sha1(b).digest()
 
+    def test_extra_algorithms(self):
+        expected_results = {
+            "md5": "bb649c83dd1ea5c9d9dec9a18df0ffe9",
+            "md4": "c275b8454684ea416b93d7a418b43176",
+            "mdc2": None,   # XXX find the correct expected value
+            "sha": "e2b0a8609b47c58e5d984c9ccfe69f9b654b032b",
+            "ripemd160": "cc4a5ce1b3df48aec5d22d1f16b894a0b894eccc",
+            "whirlpool": ("1a22b79fe5afda02c63a25927193ed01dc718b74"
+                          "026e597608ce431f9c3d2c9e74a7350b7fbb7c5d"
+                          "4effe5d7a31879b8b7a10fd2f544c4ca268ecc6793923583"),
+            }
+        import _hashlib
+        test_string = "Nobody inspects the spammish repetition"
+        for hash_name, expected in sorted(expected_results.items()):
+            try:
+                m = _hashlib.new(hash_name)
+            except ValueError, e:
+                print 'skipped %s: %s' % (hash_name, e)
+                continue
+            m.update(test_string)
+            got = m.hexdigest()
+            assert got and type(got) is str and len(got) % 2 == 0
+            got.decode('hex')
+            if expected is not None:
+                assert got == expected

pypy/module/cpyext/api.py

         elif sys.platform.startswith('linux'):
             compile_extra.append("-Werror=implicit-function-declaration")
         export_symbols_eci.append('pypyAPI')
+        compile_extra.append('-g')
     else:
         kwds["includes"] = ['Python.h'] # this is our Python.h
 

pypy/module/imp/importing.py

     # XXX Check for frozen modules?
     #     when w_path is a string
 
+    delayed_builtin = None
+    w_lib_extensions = None
+
     if w_path is None:
         # check the builtin modules
         if modulename in space.builtin_modules:
-            return FindInfo(C_BUILTIN, modulename, None)
+            delayed_builtin = FindInfo(C_BUILTIN, modulename, None)
+            # a "real builtin module xx" shadows every file "xx.py" there
+            # could possibly be; a "pseudo-extension module" does not, and
+            # is only loaded at the point in sys.path where we find
+            # '.../lib_pypy/__extensions__'.
+            if modulename in space.MODULES_THAT_ALWAYS_SHADOW:
+                return delayed_builtin
+            w_lib_extensions = space.sys.get_state(space).w_lib_extensions
         w_path = space.sys.get('path')
 
     # XXX check frozen modules?
     if w_path is not None:
         for w_pathitem in space.unpackiterable(w_path):
             # sys.path_hooks import hook
+            if (w_lib_extensions is not None and
+                    space.eq_w(w_pathitem, w_lib_extensions)):
+                return delayed_builtin
             if use_loader:
                 w_loader = find_in_path_hooks(space, w_modulename, w_pathitem)
                 if w_loader:
                        # Out of file descriptors.
 
     # not found
-    return None
+    return delayed_builtin
 
 def _prepare_module(space, w_mod, filename, pkgdir):
     w = space.wrap

pypy/module/imp/test/test_import.py

                     test_reload = "def test():\n    raise ValueError\n",
                     infinite_reload = "import infinite_reload; reload(infinite_reload)",
                     del_sys_module = "import sys\ndel sys.modules['del_sys_module']\n",
+                    itertools = "hello_world = 42\n",
+                    gc = "should_never_be_seen = 42\n",
                     )
     root.ensure("notapackage", dir=1)    # empty, no __init__.py
     setuppkg("pkg",
 class AppTestImport:
 
     def setup_class(cls): # interpreter-level
+        cls.space = gettestobjspace(usemodules=['itertools'])
+        cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
         cls.saved_modules = _setup(cls.space)
         #XXX Compile class
 
         else:
             assert False, 'should not work'
 
+    def test_shadow_builtin(self):
+        if self.runappdirect: skip("hard to test: module is already imported")
+        # 'import gc' is supposed to always find the built-in module;
+        # like CPython, it is a built-in module, so it shadows everything,
+        # even though there is a gc.py.
+        import sys
+        assert 'gc' not in sys.modules
+        import gc
+        assert not hasattr(gc, 'should_never_be_seen')
+        assert '(built-in)' in repr(gc)
+        del sys.modules['gc']
+
+    def test_shadow_extension_1(self):
+        if self.runappdirect: skip("hard to test: module is already imported")
+        # 'import itertools' is supposed to find itertools.py if there is
+        # one in sys.path.
+        import sys
+        assert 'itertools' not in sys.modules
+        import itertools
+        assert hasattr(itertools, 'hello_world')
+        assert not hasattr(itertools, 'count')
+        assert '(built-in)' not in repr(itertools)
+        del sys.modules['itertools']
+
+    def test_shadow_extension_2(self):
+        if self.runappdirect: skip("hard to test: module is already imported")
+        # 'import itertools' is supposed to find the built-in module even
+        # if there is also one in sys.path as long as it is *after* the
+        # special entry '.../lib_pypy/__extensions__'.  (Note that for now
+        # there is one in lib_pypy/itertools.py, which should not be seen
+        # either; hence the (built-in) test below.)
+        import sys
+        assert 'itertools' not in sys.modules
+        sys.path.append(sys.path.pop(0))
+        try:
+            import itertools
+            assert not hasattr(itertools, 'hello_world')
+            assert hasattr(itertools, 'izip')
+            assert '(built-in)' in repr(itertools)
+        finally:
+            sys.path.insert(0, sys.path.pop())
+        del sys.modules['itertools']
+
+
 class TestAbi:
     def test_abi_tag(self):
         space1 = gettestobjspace(soabi='TEST')

pypy/module/micronumpy/__init__.py

         'number': 'interp_boxes.W_NumberBox',
         'integer': 'interp_boxes.W_IntegerBox',
         'signedinteger': 'interp_boxes.W_SignedIntegerBox',
+        'unsignedinteger': 'interp_boxes.W_UnsignedIntegerBox',
         'bool_': 'interp_boxes.W_BoolBox',
         'int8': 'interp_boxes.W_Int8Box',
+        'uint8': 'interp_boxes.W_UInt8Box',
         'int16': 'interp_boxes.W_Int16Box',
+        'uint16': 'interp_boxes.W_UInt16Box',
         'int32': 'interp_boxes.W_Int32Box',
+        'uint32': 'interp_boxes.W_UInt32Box',
         'int64': 'interp_boxes.W_Int64Box',
+        'uint64': 'interp_boxes.W_UInt64Box',
         'int_': 'interp_boxes.W_LongBox',
         'inexact': 'interp_boxes.W_InexactBox',
         'floating': 'interp_boxes.W_FloatingBox',

pypy/module/micronumpy/compile.py

     w_ValueError = None
     w_TypeError = None
     w_IndexError = None
+    w_OverflowError = None
     w_None = None
 
     w_bool = "bool"
         # XXX array probably
         assert False
 
+    def exception_match(self, w_exc_type, w_check_class):
+        # Good enough for now
+        raise NotImplementedError
+
 class FloatObject(W_Root):
     tp = FakeSpace.w_float
     def __init__(self, floatval):

pypy/module/micronumpy/interp_boxes.py

     descr_neg = _unaryop_impl("negative")
     descr_abs = _unaryop_impl("absolute")
 
+    def descr_tolist(self, space):
+        return self.get_dtype(space).itemtype.to_builtin_type(space, self)
+
 
 class W_BoolBox(W_GenericBox, PrimitiveBox):
     descr__new__, get_dtype = new_dtype_getter("bool")
 class W_SignedIntegerBox(W_IntegerBox):
     pass
 
-class W_UnsignedIntgerBox(W_IntegerBox):
+class W_UnsignedIntegerBox(W_IntegerBox):
     pass
 
 class W_Int8Box(W_SignedIntegerBox, PrimitiveBox):
     descr__new__, get_dtype = new_dtype_getter("int8")
 
-class W_UInt8Box(W_UnsignedIntgerBox, PrimitiveBox):
+class W_UInt8Box(W_UnsignedIntegerBox, PrimitiveBox):
     descr__new__, get_dtype = new_dtype_getter("uint8")
 
 class W_Int16Box(W_SignedIntegerBox, PrimitiveBox):
     descr__new__, get_dtype = new_dtype_getter("int16")
 
-class W_UInt16Box(W_UnsignedIntgerBox, PrimitiveBox):
+class W_UInt16Box(W_UnsignedIntegerBox, PrimitiveBox):
     descr__new__, get_dtype = new_dtype_getter("uint16")
 
 class W_Int32Box(W_SignedIntegerBox, PrimitiveBox):
     descr__new__, get_dtype = new_dtype_getter("int32")
 
-class W_UInt32Box(W_UnsignedIntgerBox, PrimitiveBox):
+class W_UInt32Box(W_UnsignedIntegerBox, PrimitiveBox):
     descr__new__, get_dtype = new_dtype_getter("uint32")
 
 class W_LongBox(W_SignedIntegerBox, PrimitiveBox):
     descr__new__, get_dtype = new_dtype_getter("long")
 
-class W_ULongBox(W_UnsignedIntgerBox, PrimitiveBox):
+class W_ULongBox(W_UnsignedIntegerBox, PrimitiveBox):
     pass
 
 class W_Int64Box(W_SignedIntegerBox, PrimitiveBox):
     descr__new__, get_dtype = new_dtype_getter("int64")
 
-class W_UInt64Box(W_UnsignedIntgerBox, PrimitiveBox):
-    pass
+class W_UInt64Box(W_UnsignedIntegerBox, PrimitiveBox):
+    descr__new__, get_dtype = new_dtype_getter("uint64")
 
 class W_InexactBox(W_NumberBox):
     _attrs_ = ()
 
     __neg__ = interp2app(W_GenericBox.descr_neg),
     __abs__ = interp2app(W_GenericBox.descr_abs),
+
+    tolist = interp2app(W_GenericBox.descr_tolist),
 )
 
 W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
     __module__ = "numpypy",
 )
 
+W_UnsignedIntegerBox.typedef = TypeDef("unsignedinteger", W_IntegerBox.typedef,
+    __module__ = "numpypy",
+)
+
 W_Int8Box.typedef = TypeDef("int8", W_SignedIntegerBox.typedef,
     __module__ = "numpypy",
     __new__ = interp2app(W_Int8Box.descr__new__.im_func),
 )
 
-W_UInt8Box.typedef = TypeDef("uint8", W_UnsignedIntgerBox.typedef,
+W_UInt8Box.typedef = TypeDef("uint8", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
+    __new__ = interp2app(W_UInt8Box.descr__new__.im_func),
 )
 
 W_Int16Box.typedef = TypeDef("int16", W_SignedIntegerBox.typedef,
     __new__ = interp2app(W_Int16Box.descr__new__.im_func),
 )
 
-W_UInt16Box.typedef = TypeDef("uint16", W_UnsignedIntgerBox.typedef,
+W_UInt16Box.typedef = TypeDef("uint16", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
+    __new__ = interp2app(W_UInt16Box.descr__new__.im_func),
 )
 
 W_Int32Box.typedef = TypeDef("int32", W_SignedIntegerBox.typedef,
     __new__ = interp2app(W_Int32Box.descr__new__.im_func),
 )
 
-W_UInt32Box.typedef = TypeDef("uint32", W_UnsignedIntgerBox.typedef,
+W_UInt32Box.typedef = TypeDef("uint32", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
+    __new__ = interp2app(W_UInt32Box.descr__new__.im_func),
 )
 
 if LONG_BIT == 32:
     __module__ = "numpypy",
 )
 
-W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntgerBox.typedef,
+W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
 )
 
     __new__ = interp2app(W_Int64Box.descr__new__.im_func),
 )
 
-W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntgerBox.typedef,
+W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef,
     __module__ = "numpypy",
+    __new__ = interp2app(W_UInt64Box.descr__new__.im_func),
 )
 
 W_InexactBox.typedef = TypeDef("inexact", W_NumberBox.typedef,

pypy/module/micronumpy/interp_numarray.py

             arr.setshape(space, new_shape)
         return arr
 
+    def descr_tolist(self, space):
+        if len(self.shape) == 0:
+            assert isinstance(self, Scalar)
+            return self.value.descr_tolist(space)
+        w_result = space.newlist([])
+        for i in range(self.shape[0]):
+            space.call_method(w_result, "append",
+                space.call_method(self.descr_getitem(space, space.wrap(i)), "tolist")
+            )
+        return w_result
+
     def descr_mean(self, space):
         return space.div(self.descr_sum(space), space.wrap(self.find_size()))
 
 
     copy = interp2app(BaseArray.descr_copy),
     reshape = interp2app(BaseArray.descr_reshape),
+    tolist = interp2app(BaseArray.descr_tolist),
 )
 
 

pypy/module/micronumpy/test/test_dtypes.py

         raises(TypeError, numpy.integer, 0)
         exc = raises(TypeError, numpy.signedinteger, 0)
         assert str(exc.value) == "cannot create 'signedinteger' instances"
+        exc = raises(TypeError, numpy.unsignedinteger, 0)
+        assert str(exc.value) == "cannot create 'unsignedinteger' instances"
 
         raises(TypeError, numpy.floating, 0)
         raises(TypeError, numpy.inexact, 0)
         assert type(int(x)) is int
         assert int(x) == -128
 
+    def test_uint8(self):
+        import numpypy as numpy
+
+        assert numpy.uint8.mro() == [numpy.uint8, numpy.unsignedinteger, numpy.integer, numpy.number, numpy.generic, object]
+
+        a = numpy.array([1, 2, 3], numpy.uint8)
+        assert type(a[1]) is numpy.uint8
+        assert numpy.dtype("uint8").type is numpy.uint8
+
+        x = numpy.uint8(128)
+        assert x == 128
+        assert x != -128
+        assert type(x) is numpy.uint8
+        assert repr(x) == "128"
+
+        assert type(int(x)) is int
+        assert int(x) == 128
+
+        assert numpy.uint8(255) == 255
+        assert numpy.uint8(256) == 0
+
     def test_int16(self):
         import numpypy as numpy
 
         x = numpy.int16(3)
         assert x == 3
+        assert numpy.int16(32767) == 32767
+        assert numpy.int16(32768) == -32768
+
+    def test_uint16(self):
+        import numpypy as numpy
+
+        assert numpy.uint16(65535) == 65535
+        assert numpy.uint16(65536) == 0
 
     def test_int32(self):
         import numpypy as numpy
 
         x = numpy.int32(23)
         assert x == 23
+        assert numpy.int32(2147483647) == 2147483647
+        assert numpy.int32(2147483648) == -2147483648
+
+    def test_uint32(self):
+        import numpypy as numpy
+
+        assert numpy.uint32(4294967295) == 4294967295
+        assert numpy.uint32(4294967296) == 0
 
     def test_int_(self):
         import numpypy as numpy
         assert numpy.dtype(numpy.int64).type is numpy.int64
         assert numpy.int64(3) == 3
 
+        assert numpy.int64(9223372036854775807) == 9223372036854775807
+        raises(OverflowError, numpy.int64, 9223372036854775808)
+
+    def test_uint64(self):
+        import sys
+        import numpypy as numpy
+
+        assert numpy.uint64.mro() == [numpy.uint64, numpy.unsignedinteger, numpy.integer, numpy.number, numpy.generic, object]
+
+        assert numpy.dtype(numpy.uint64).type is numpy.uint64
+        skip("see comment")
+        # These tests pass "by chance" on numpy, things that are larger than
+        # platform long (i.e. a python int), don't get put in a normal box,
+        # instead they become an object array containing a long, we don't have
+        # yet, so these can't pass.
+        assert numpy.uint64(9223372036854775808) == 9223372036854775808
+        assert numpy.uint64(18446744073709551615) == 18446744073709551615
+        raises(OverflowError, numpy.uint64(18446744073709551616))
+
     def test_float32(self):
         import numpypy as numpy
 

pypy/module/micronumpy/test/test_numarray.py

         b[0] = 3
         assert b.__debug_repr__() == 'Call2(add, forced=Array)'
 
+    def test_tolist_scalar(self):
+        from numpypy import int32, bool_
+        x = int32(23)
+        assert x.tolist() == 23
+        assert type(x.tolist()) is int
+        y = bool_(True)
+        assert y.tolist() is True
+
+    def test_tolist_zerodim(self):
+        from numpypy import array
+        x = array(3)
+        assert x.tolist() == 3
+        assert type(x.tolist()) is int
+
+    def test_tolist_singledim(self):
+        from numpypy import array
+        a = array(range(5))
+        assert a.tolist() == [0, 1, 2, 3, 4]
+        assert type(a.tolist()[0]) is int
+        b = array([0.2, 0.4, 0.6])
+        assert b.tolist() == [0.2, 0.4, 0.6]
+
+    def test_tolist_multidim(self):
+        from numpypy import array
+        a = array([[1, 2], [3, 4]])
+        assert a.tolist() == [[1, 2], [3, 4]]
+
+    def test_tolist_view(self):
+        from numpypy import array
+        a = array([[1,2],[3,4]])
+        assert (a + a).tolist() == [[2, 4], [6, 8]]
+
+    def test_tolist_slice(self):
+        from numpypy import array
+        a = array([[17.1, 27.2], [40.3, 50.3]])
+        assert a[:,0].tolist() == [17.1, 40.3]
+        assert a[0].tolist() == [17.1, 27.2]
+
+
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import numpypy

pypy/module/micronumpy/test/test_zjit.py

         # sure it was optimized correctly.
         # XXX the comment above is wrong now.  We need preferrably a way to
         # count the two loops separately
-        self.check_resops({'setinteriorfield_raw': 4, 'guard_nonnull': 1, 'getfield_gc': 41,
+        self.check_resops({'setinteriorfield_raw': 4, 'guard_nonnull': 1,
+                           'getfield_gc': 35, 'getfield_gc_pure': 6,
                            'guard_class': 22, 'int_add': 8, 'float_mul': 2,
                            'guard_isnull': 2, 'jump': 4, 'int_ge': 4,
                            'getinteriorfield_raw': 4, 'float_add': 2, 'guard_false': 4,

pypy/module/micronumpy/types.py

 import functools
 import math
 
+from pypy.interpreter.error import OperationError
 from pypy.module.micronumpy import interp_boxes
 from pypy.objspace.std.floatobject import float2string
 from pypy.rlib import rfloat, libffi, clibffi
         w_obj.__init__(self._coerce(space, w_item).value)
         return w_obj
 
+    def to_builtin_type(self, space, box):
+        return space.wrap(self.unbox(box))
+
     def _coerce(self, space, w_item):
         raise NotImplementedError
 
     T = rffi.ULONGLONG
     BoxType = interp_boxes.W_UInt64Box
 
+    def _coerce(self, space, w_item):
+        try:
+            return Integer._coerce(self, space, w_item)
+        except OperationError, e:
+            if not e.match(space, space.w_OverflowError):
+                raise
+        bigint = space.bigint_w(w_item)
+        try:
+            value = bigint.toulonglong()
+        except OverflowError:
+            raise OperationError(space.w_OverflowError, space.w_None)
+        return self.box(value)
+
 class Float(Primitive):
     _mixin_ = True
 

pypy/module/sys/__init__.py

     def get_flag(self, name):
         space = self.space
         return space.int_w(space.getattr(self.get('flags'), space.wrap(name)))
+
+    def get_state(self, space):
+        from pypy.module.sys import state
+        return state.get(space)

pypy/module/sys/state.py

         # Initialize the default path
         pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
         srcdir = os.path.dirname(pypydir)
-        path = getinitialpath(srcdir)
+        path = getinitialpath(self, srcdir)
         self.w_path = space.newlist([space.wrap(p) for p in path])
 
 def checkdir(path):
 
 platform = sys.platform
 
-def getinitialpath(prefix):
+def getinitialpath(state, prefix):
     from pypy.module.sys.version import CPYTHON_VERSION
     dirname = '%d.%d' % (CPYTHON_VERSION[0],
                          CPYTHON_VERSION[1])
     checkdir(lib_pypy)
 
     importlist = []
+    #
+    if state is not None:    # 'None' for testing only
+        lib_extensions = os.path.join(lib_pypy, '__extensions__')
+        state.w_lib_extensions = state.space.wrap(lib_extensions)
+        importlist.append(lib_extensions)
+    #
     importlist.append(lib_pypy)
     importlist.append(python_std_lib_modified)
     importlist.append(python_std_lib)
 @unwrap_spec(srcdir=str)
 def pypy_initial_path(space, srcdir):
     try:
-        path = getinitialpath(srcdir)
+        path = getinitialpath(get(space), srcdir)
     except OSError:
         return space.w_None
     else:

pypy/module/sys/test/test_initialpath.py

 
 def test_stdlib_in_prefix(tmpdir):
     dirs = build_hierarchy(tmpdir)
-    path = getinitialpath(str(tmpdir))
+    path = getinitialpath(None, str(tmpdir))
     # we get at least 'dirs', and maybe more (e.g. plat-linux2)
     assert path[:len(dirs)] == map(str, dirs)
 
     lib_pypy, lib_python_modified, lib_python = build_hierarchy(tmpdir)
     lib_tk_modified = lib_python_modified.join('lib-tk')
     lib_tk = lib_python.join('lib-tk')
-    path = getinitialpath(str(tmpdir))
+    path = getinitialpath(None, str(tmpdir))
     i = path.index(str(lib_tk_modified))
     j = path.index(str(lib_tk))
     assert i < j

pypy/objspace/fake/objspace.py

         t = TranslationContext(config=config)
         self.t = t     # for debugging
         ann = t.buildannotator()
+        ann.policy.allow_someobjects = False
         if func is not None:
             ann.build_types(func, argtypes, complete_now=False)
         #

pypy/rlib/rdynload.py

 if _WIN32:
     DLLHANDLE = rwin32.HMODULE
 
-    def dlopen(name):
+    def dlopen(name, mode=-1):
+        # mode is unused on windows, but a consistant signature
         res = rwin32.LoadLibrary(name)
         if not res:
             err = rwin32.GetLastError()

pypy/rlib/ropenssl.py

         'struct GENERAL_NAME_st',
         [('type', rffi.INT),
          ])
+    EVP_MD_st = rffi_platform.Struct(
+        'EVP_MD',
+        [('md_size', rffi.INT),
+         ('block_size', rffi.INT)])
     EVP_MD_SIZE = rffi_platform.SizeOf('EVP_MD')
     EVP_MD_CTX_SIZE = rffi_platform.SizeOf('EVP_MD_CTX')
 
              [BIO, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP], X509)
 
 EVP_MD_CTX = rffi.COpaquePtr('EVP_MD_CTX', compilation_info=eci)
-EVP_MD     = rffi.COpaquePtr('EVP_MD', compilation_info=eci)
+EVP_MD     = lltype.Ptr(EVP_MD_st)
 
 OpenSSL_add_all_digests = external(
     'OpenSSL_add_all_digests', [], lltype.Void)

pypy/rpython/memory/gc/minimark.py

 
 Environment variables can be used to fine-tune the following parameters:
 
- PYPY_GC_NURSERY        The nursery size.  Defaults to half the size of
-                        the L2 cache.  Try values like '1.2MB'.  Small values
+ PYPY_GC_NURSERY        The nursery size.  Defaults to '4MB'.  Small values
                         (like 1 or 1KB) are useful for debugging.
 
  PYPY_GC_MAJOR_COLLECT  Major collection memory factor.  Default is '1.82',
 #
 #  * young objects: allocated in the nursery if they are not too large, or
 #    raw-malloced otherwise.  The nursery is a fixed-size memory buffer of
-#    half the size of the L2 cache.  When full, we do a minor collection;
+#    4MB by default.  When full, we do a minor collection;
 #    the surviving objects from the nursery are moved outside, and the
 #    non-surviving raw-malloced objects are freed.  All surviving objects
 #    become old.
             # size (needed to handle mallocs just below 'large_objects') but
             # hacking at the current nursery position in collect_and_reserve().
             if newsize <= 0:
-                newsize = env.estimate_best_nursery_size()
+                newsize = 4*1024*1024   # fixed to 4MB by default
+                #        (it was env.estimate_best_nursery_size())
                 if newsize <= 0:
                     newsize = defaultsize
             if newsize < minsize:

pypy/translator/goal/app_main.py

     def pypy_initial_path(s):
         from pypy.module.sys.state import getinitialpath
         try:
-            return getinitialpath(s)
+            return getinitialpath(None, s)
         except OSError:
             return None
 

pypy/translator/goal/test2/test_app_main.py

             newpath = app_main.get_library_path('/tmp/pypy-c') # stdlib not found
             assert newpath == sys.path
             newpath = app_main.get_library_path(self.fake_exe)
+            if newpath[0].endswith('__extensions__'):
+                newpath = newpath[1:]
             # we get at least 'expected_path', and maybe more (e.g.plat-linux2)
             assert newpath[:len(self.expected_path)] == self.expected_path
         finally:

pypy/translator/sandbox/sandlib.py

 
 import py
 import sys, os, posixpath, errno, stat, time
-from pypy.rpython.module.ll_os_stat import s_StatResult
 from pypy.tool.ansi_print import AnsiLog
-from pypy.rlib.rarithmetic import r_longlong
 import subprocess
 from pypy.tool.killsubprocess import killsubprocess
+from pypy.translator.sandbox.vfs import UID, GID
 
 class MyAnsiLog(AnsiLog):
     KW_TO_COLOR = {
 from pypy.tool.lib_pypy import import_from_lib_pypy
 marshal = import_from_lib_pypy('marshal')
 
+# Non-marshal result types
+RESULTTYPE_STATRESULT = object()
+RESULTTYPE_LONGLONG = object()
+
 def read_message(f, timeout=None):
     # warning: 'timeout' is not really reliable and should only be used
     # for testing.  Also, it doesn't work if the file f does any buffering.
             marshal.dump(msg, g)
         else:
             marshal.dump(msg, g, 0)
+    elif resulttype is RESULTTYPE_STATRESULT:
+        # Hand-coded marshal for stat results that mimics what rmarshal expects.
+        # marshal.dump(tuple(msg)) would have been too easy. rmarshal insists
+        # on 64-bit ints at places, even when the value fits in 32 bits.
+        import struct
+        st = tuple(msg)
+        fmt = "iIIiiiIfff"
+        buf = []
+        buf.append(struct.pack("<ci", '(', len(st)))
+        for c, v in zip(fmt, st):
+            if c == 'i':
+                buf.append(struct.pack("<ci", c, v))
+            elif c == 'I':
+                buf.append(struct.pack("<cq", c, v))
+            elif c == 'f':
+                fstr = "%g" % v
+                buf.append(struct.pack("<cB", c, len(fstr)))
+                buf.append(fstr)
+        g.write(''.join(buf))
+    elif resulttype is RESULTTYPE_LONGLONG:
+        import struct
+        g.write(struct.pack("<cq", 'I', msg))
     else:
-        # use the exact result type for encoding
-        from pypy.rlib.rmarshal import get_marshaller
-        buf = []
-        get_marshaller(resulttype)(buf, msg)
-        g.write(''.join(buf))
+        raise Exception("Can't marshal: %r (%r)" % (msg, resulttype))
 
 # keep the table in sync with rsandbox.reraise_error()
 EXCEPTION_TABLE = [
     def __init__(self, *args, **kwds):
         super(VirtualizedSandboxedProc, self).__init__(*args, **kwds)
         self.virtual_root = self.build_virtual_root()
-        self.open_fds = {}   # {virtual_fd: real_file_object}
+        self.open_fds = {}   # {virtual_fd: (real_file_object, node)}
 
     def build_virtual_root(self):
         raise NotImplementedError("must be overridden")
     def do_ll_os__ll_os_stat(self, vpathname):
         node = self.get_node(vpathname)
         return node.stat()
-    do_ll_os__ll_os_stat.resulttype = s_StatResult
+    do_ll_os__ll_os_stat.resulttype = RESULTTYPE_STATRESULT
 
     do_ll_os__ll_os_lstat = do_ll_os__ll_os_stat
 
     def do_ll_os__ll_os_isatty(self, fd):
         return self.virtual_console_isatty and fd in (0, 1, 2)
 
-    def allocate_fd(self, f):
+    def allocate_fd(self, f, node=None):
         for fd in self.virtual_fd_range:
             if fd not in self.open_fds:
-                self.open_fds[fd] = f
+                self.open_fds[fd] = (f, node)
                 return fd
         else:
             raise OSError(errno.EMFILE, "trying to open too many files")
 
-    def get_file(self, fd):
+    def get_fd(self, fd, throw=True):
+        """Get the objects implementing file descriptor `fd`.
+
+        Returns a pair, (open file, vfs node)
+
+        `throw`: if true, raise OSError for bad fd, else return (None, None).
+        """
         try:
-            return self.open_fds[fd]
+            f, node = self.open_fds[fd]
         except KeyError:
-            raise OSError(errno.EBADF, "bad file descriptor")
+            if throw:
+                raise OSError(errno.EBADF, "bad file descriptor")
+            return None, None
+        return f, node
+
+    def get_file(self, fd, throw=True):
+        """Return the open file for file descriptor `fd`."""
+        return self.get_fd(fd, throw)[0]
 
     def do_ll_os__ll_os_open(self, vpathname, flags, mode):
         node = self.get_node(vpathname)
             raise OSError(errno.EPERM, "write access denied")
         # all other flags are ignored
         f = node.open()
-        return self.allocate_fd(f)
+        return self.allocate_fd(f, node)
 
     def do_ll_os__ll_os_close(self, fd):
         f = self.get_file(fd)
         f.close()
 
     def do_ll_os__ll_os_read(self, fd, size):
-        try:
-            f = self.open_fds[fd]
-        except KeyError:
+        f = self.get_file(fd, throw=False)
+        if f is None:
             return super(VirtualizedSandboxedProc, self).do_ll_os__ll_os_read(
                 fd, size)
         else:
             # don't try to read more than 256KB at once here
             return f.read(min(size, 256*1024))
 
+    def do_ll_os__ll_os_fstat(self, fd):
+        f, node = self.get_fd(fd)
+        return node.stat()
+    do_ll_os__ll_os_fstat.resulttype = RESULTTYPE_STATRESULT
+
     def do_ll_os__ll_os_lseek(self, fd, pos, how):
         f = self.get_file(fd)
         f.seek(pos, how)
         return f.tell()
-    do_ll_os__ll_os_lseek.resulttype = r_longlong
+    do_ll_os__ll_os_lseek.resulttype = RESULTTYPE_LONGLONG
 
     def do_ll_os__ll_os_getcwd(self):
         return self.virtual_cwd
         node = self.get_node(vpathname)
         return node.keys()
 
+    def do_ll_os__ll_os_getuid(self):
+        return UID
+    do_ll_os__ll_os_geteuid = do_ll_os__ll_os_getuid
+
+    def do_ll_os__ll_os_getgid(self):
+        return GID
+    do_ll_os__ll_os_getegid = do_ll_os__ll_os_getgid
+
 
 class VirtualizedSocketProc(VirtualizedSandboxedProc):
     """ Extends VirtualizedSandboxProc with socket
 
     def do_ll_os__ll_os_read(self, fd, size):
         if fd in self.sockets:
-            return self.open_fds[fd].recv(size)
+            return self.get_file(fd).recv(size)
         return super(VirtualizedSocketProc, self).do_ll_os__ll_os_read(
             fd, size)
 
     def do_ll_os__ll_os_write(self, fd, data):
         if fd in self.sockets:
-            return self.open_fds[fd].send(data)
+            return self.get_file(fd).send(data)
         return super(VirtualizedSocketProc, self).do_ll_os__ll_os_write(
             fd, data)
 

pypy/translator/sandbox/test/test_sandbox.py

     assert tail == ""
 
 def test_stat_ftruncate():
-    from pypy.rpython.module.ll_os_stat import s_StatResult
+    from pypy.translator.sandbox.sandlib import RESULTTYPE_STATRESULT
     from pypy.rlib.rarithmetic import r_longlong
     r0x12380000007 = r_longlong(0x12380000007)
 
     g, f = os.popen2(exe, "t", 0)
     st = os.stat_result((55, 0, 0, 0, 0, 0, 0x12380000007, 0, 0, 0))
     expect(f, g, "ll_os.ll_os_stat", ("somewhere",), st,
-           resulttype = s_StatResult)
+           resulttype = RESULTTYPE_STATRESULT)
     expect(f, g, "ll_os.ll_os_ftruncate", (55, 0x12380000007), None)
     g.close()
     tail = f.read()

pypy/translator/sandbox/test/test_sandlib.py

 import py
-import os, StringIO
+import errno, os, StringIO
 from pypy.tool.sourcetools import func_with_new_name
 from pypy.rpython.lltypesystem import rffi
 from pypy.translator.sandbox.sandlib import SandboxedProc
 from pypy.translator.sandbox.sandlib import SimpleIOSandboxedProc
+from pypy.translator.sandbox.sandlib import VirtualizedSandboxedProc
 from pypy.translator.sandbox.sandlib import VirtualizedSocketProc
 from pypy.translator.sandbox.test.test_sandbox import compile
+from pypy.translator.sandbox.vfs import Dir, File, RealDir, RealFile
 
 
-class MySandboxedProc(SandboxedProc):
+class MockSandboxedProc(SandboxedProc):
+    """A sandbox process wrapper that replays expected syscalls."""
 
     def __init__(self, args, expected):
         SandboxedProc.__init__(self, args)
         return 0
     exe = compile(entry_point)
 
-    proc = MySandboxedProc([exe, 'x1', 'y2'], expected = [
+    proc = MockSandboxedProc([exe, 'x1', 'y2'], expected = [
         ("open", ("/tmp/foobar", os.O_RDONLY, 0777), 77),
         ("read", (77, 123), "he\x00llo"),
         ("write", (77, "world\x00!\x00"), 42),
         return n
     exe = compile(entry_point)
 
-    proc = MySandboxedProc([exe, 'spam', 'egg'], expected = [
+    proc = MockSandboxedProc([exe, 'spam', 'egg'], expected = [
         ("foobar", ("spam",), 2),
         ("foobar", ("egg",), 0),
         ])
         return 0
     exe = compile(entry_point)
 
-    proc = MySandboxedProc([exe], expected = [
+    proc = MockSandboxedProc([exe], expected = [
         ("open", ("/tmp/foobar", os.O_RDONLY, 0777), OSError(-42, "baz")),
         ("close", (-42,), None),
         ])
     proc.handle_forever()
     assert proc.seen == len(proc.expected)
+
+
+class SandboxedProcWithFiles(VirtualizedSandboxedProc, SimpleIOSandboxedProc):
+    """A sandboxed process with a simple virtualized filesystem.
+
+    For testing file operations.
+
+    """
+    def build_virtual_root(self):
+        return Dir({
+            'hi.txt': File("Hello, world!\n"),
+            'this.pyc': RealFile(__file__),
+             })
+
+def test_too_many_opens():
+    def entry_point(argv):
+        try:
+            open_files = []
+            for i in range(500):
+                fd = os.open('/hi.txt', os.O_RDONLY, 0777)
+                open_files.append(fd)
+                txt = os.read(fd, 100)
+                if txt != "Hello, world!\n":
+                    print "Wrong content: %s" % txt
+        except OSError, e:
+            # We expect to get EMFILE, for opening too many files.
+            if e.errno != errno.EMFILE:
+                print "OSError: %s!" % (e.errno,)
+        else:
+            print "We opened 500 fake files! Shouldn't have been able to."
+
+        for fd in open_files:
+            os.close(fd)
+
+        try:
+            open_files = []
+            for i in range(500):
+                fd = os.open('/this.pyc', os.O_RDONLY, 0777)
+                open_files.append(fd)
+        except OSError, e:
+            # We expect to get EMFILE, for opening too many files.
+            if e.errno != errno.EMFILE:
+                print "OSError: %s!" % (e.errno,)
+        else:
+            print "We opened 500 real files! Shouldn't have been able to."
+
+        print "All ok!"
+        return 0
+    exe = compile(entry_point)
+
+    proc = SandboxedProcWithFiles([exe])
+    output, error = proc.communicate("")
+    assert output == "All ok!\n"
+    assert error == ""
+
+def test_fstat():
+    def compare(a, b, i):
+        if a != b:
+            print "stat and fstat differ @%d: %s != %s" % (i, a, b)
+
+    def entry_point(argv):
+        try:
+            # Open a file, and compare stat and fstat
+            fd = os.open('/hi.txt', os.O_RDONLY, 0777)
+            st = os.stat('/hi.txt')
+            fs = os.fstat(fd)
+            # RPython requires the index for stat to be a constant.. :(
+            compare(st[0], fs[0], 0)
+            compare(st[1], fs[1], 1)
+            compare(st[2], fs[2], 2)
+            compare(st[3], fs[3], 3)
+            compare(st[4], fs[4], 4)
+            compare(st[5], fs[5], 5)
+            compare(st[6], fs[6], 6)
+            compare(st[7], fs[7], 7)
+            compare(st[8], fs[8], 8)
+            compare(st[9], fs[9], 9)
+        except OSError, e:
+            print "OSError: %s" % (e.errno,)
+        print "All ok!"
+        return 0
+    exe = compile(entry_point)
+
+    proc = SandboxedProcWithFiles([exe])
+    output, error = proc.communicate("")
+    assert output == "All ok!\n"
+    assert error == ""
+
+def test_lseek():
+    def char_should_be(c, should):
+        if c != should:
+            print "Wrong char: '%s' should be '%s'" % (c, should)
+
+    def entry_point(argv):
+        fd = os.open('/hi.txt', os.O_RDONLY, 0777)
+        char_should_be(os.read(fd, 1), "H")
+        new = os.lseek(fd, 3, os.SEEK_CUR)
+        if new != 4:
+            print "Wrong offset, %d should be 4" % new
+        char_should_be(os.read(fd, 1), "o")
+        new = os.lseek(fd, -3, os.SEEK_END)
+        if new != 11:
+            print "Wrong offset, %d should be 11" % new
+        char_should_be(os.read(fd, 1), "d")
+        new = os.lseek(fd, 7, os.SEEK_SET)
+        if new != 7:
+            print "Wrong offset, %d should be 7" % new
+        char_should_be(os.read(fd, 1), "w")
+        print "All ok!"
+        return 0
+    exe = compile(entry_point)
+
+    proc = SandboxedProcWithFiles([exe])
+    output, error = proc.communicate("")
+    assert output == "All ok!\n"
+    assert error == ""
+
+def test_getuid():
+    def entry_point(argv):
+        import os
+        print "uid is %s" % os.getuid()
+        print "euid is %s" % os.geteuid()
+        print "gid is %s" % os.getgid()
+        print "egid is %s" % os.getegid()
+        return 0
+    exe = compile(entry_point)
+
+    proc = SandboxedProcWithFiles([exe])
+    output, error = proc.communicate("")
+    assert output == "uid is 1000\neuid is 1000\ngid is 1000\negid is 1000\n"
+    assert error == ""