Commits

Justin Peel committed 12c31a3 Merge

merge in default

Comments (0)

Files changed (150)

dotviewer/graphparse.py

     print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
     return 'unknown'
 
-def dot2plain(content, contenttype, use_codespeak=False):
-    if contenttype == 'plain':
-        # already a .plain file
-        return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+    if contenttype != 'neato':
+        cmdline = 'dot -Tplain'
+    else:
+        cmdline = 'neato -Tplain'
+    #print >> sys.stderr, '* running:', cmdline
+    close_fds = sys.platform != 'win32'
+    p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    (child_in, child_out) = (p.stdin, p.stdout)
+    try:
+        import thread
+    except ImportError:
+        bkgndwrite(child_in, content)
+    else:
+        thread.start_new_thread(bkgndwrite, (child_in, content))
+    plaincontent = child_out.read()
+    child_out.close()
+    if not plaincontent:    # 'dot' is likely not installed
+        raise PlainParseError("no result from running 'dot'")
+    return plaincontent
 
-    if not use_codespeak:
-        if contenttype != 'neato':
-            cmdline = 'dot -Tplain'
-        else:
-            cmdline = 'neato -Tplain'
-        #print >> sys.stderr, '* running:', cmdline
-        close_fds = sys.platform != 'win32'
-        p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (child_in, child_out) = (p.stdin, p.stdout)
-        try:
-            import thread
-        except ImportError:
-            bkgndwrite(child_in, content)
-        else:
-            thread.start_new_thread(bkgndwrite, (child_in, content))
-        plaincontent = child_out.read()
-        child_out.close()
-        if not plaincontent:    # 'dot' is likely not installed
-            raise PlainParseError("no result from running 'dot'")
-    else:
-        import urllib
-        request = urllib.urlencode({'dot': content})
-        url = 'http://codespeak.net/pypy/convertdot.cgi'
-        print >> sys.stderr, '* posting:', url
-        g = urllib.urlopen(url, data=request)
-        result = []
-        while True:
-            data = g.read(16384)
-            if not data:
-                break
-            result.append(data)
-        g.close()
-        plaincontent = ''.join(result)
-        # very simple-minded way to give a somewhat better error message
-        if plaincontent.startswith('<body'):
-            raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+    import urllib
+    request = urllib.urlencode({'dot': content})
+    url = 'http://codespeak.net/pypy/convertdot.cgi'
+    print >> sys.stderr, '* posting:', url
+    g = urllib.urlopen(url, data=request)
+    result = []
+    while True:
+        data = g.read(16384)
+        if not data:
+            break
+        result.append(data)
+    g.close()
+    plaincontent = ''.join(result)
+    # very simple-minded way to give a somewhat better error message
+    if plaincontent.startswith('<body'):
+        raise Exception("the dot on codespeak has very likely crashed")
     return plaincontent
 
 def bkgndwrite(f, data):
 
 def parse_dot(graph_id, content, links={}, fixedfont=False):
     contenttype = guess_type(content)
-    try:
-        plaincontent = dot2plain(content, contenttype, use_codespeak=False)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
-    except PlainParseError:
-        # failed, retry via codespeak
-        plaincontent = dot2plain(content, contenttype, use_codespeak=True)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+    if contenttype == 'plain':
+        plaincontent = content
+    else:
+        try:
+            plaincontent = dot2plain_graphviz(content, contenttype)
+        except PlainParseError, e:
+            print e
+            # failed, retry via codespeak
+            plaincontent = dot2plain_codespeak(content, contenttype)
+    return list(parse_plain(graph_id, plaincontent, links, fixedfont))

lib-python/conftest.py

     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),

lib-python/modified-2.7/gzip.py

-"""Functions that read and write gzipped files.
-
-The user of the file doesn't have to worry about the compression,
-but random access is not allowed."""
-
-# based on Andrew Kuchling's minigzip.py distributed with the zlib module
-
-import struct, sys, time, os
-import zlib
-import io
-import __builtin__
-
-__all__ = ["GzipFile","open"]
-
-FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-
-READ, WRITE = 1, 2
-
-def write32u(output, value):
-    # The L format writes the bit pattern correctly whether signed
-    # or unsigned.
-    output.write(struct.pack("<L", value))
-
-def read32(input):
-    return struct.unpack("<I", input.read(4))[0]
-
-def open(filename, mode="rb", compresslevel=9):
-    """Shorthand for GzipFile(filename, mode, compresslevel).
-
-    The filename argument is required; mode defaults to 'rb'
-    and compresslevel defaults to 9.
-
-    """
-    return GzipFile(filename, mode, compresslevel)
-
-class GzipFile(io.BufferedIOBase):
-    """The GzipFile class simulates most of the methods of a file object with
-    the exception of the readinto() and truncate() methods.
-
-    """
-
-    myfileobj = None
-    max_read_chunk = 10 * 1024 * 1024   # 10Mb
-
-    def __init__(self, filename=None, mode=None,
-                 compresslevel=9, fileobj=None, mtime=None):
-        """Constructor for the GzipFile class.
-
-        At least one of fileobj and filename must be given a
-        non-trivial value.
-
-        The new class instance is based on fileobj, which can be a regular
-        file, a StringIO object, or any other object which simulates a file.
-        It defaults to None, in which case filename is opened to provide
-        a file object.
-
-        When fileobj is not None, the filename argument is only used to be
-        included in the gzip file header, which may includes the original
-        filename of the uncompressed file.  It defaults to the filename of
-        fileobj, if discernible; otherwise, it defaults to the empty string,
-        and in this case the original filename is not included in the header.
-
-        The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
-        depending on whether the file will be read or written.  The default
-        is the mode of fileobj if discernible; otherwise, the default is 'rb'.
-        Be aware that only the 'rb', 'ab', and 'wb' values should be used
-        for cross-platform portability.
-
-        The compresslevel argument is an integer from 1 to 9 controlling the
-        level of compression; 1 is fastest and produces the least compression,
-        and 9 is slowest and produces the most compression.  The default is 9.
-
-        The mtime argument is an optional numeric timestamp to be written
-        to the stream when compressing.  All gzip compressed streams
-        are required to contain a timestamp.  If omitted or None, the
-        current time is used.  This module ignores the timestamp when
-        decompressing; however, some programs, such as gunzip, make use
-        of it.  The format of the timestamp is the same as that of the
-        return value of time.time() and of the st_mtime member of the
-        object returned by os.stat().
-
-        """
-
-        # guarantee the file is opened in binary mode on platforms
-        # that care about that sort of thing
-        if mode and 'b' not in mode:
-            mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
-        if filename is None:
-            if hasattr(fileobj, 'name'): filename = fileobj.name
-            else: filename = ''
-        if mode is None:
-            if hasattr(fileobj, 'mode'): mode = fileobj.mode
-            else: mode = 'rb'
-
-        if mode[0:1] == 'r':
-            self.mode = READ
-            # Set flag indicating start of a new member
-            self._new_member = True
-            # Buffer data read from gzip file. extrastart is offset in
-            # stream where buffer starts. extrasize is number of
-            # bytes remaining in buffer from current stream position.
-            self.extrabuf = ""
-            self.extrasize = 0
-            self.extrastart = 0
-            self.name = filename
-            # Starts small, scales exponentially
-            self.min_readsize = 100
-
-        elif mode[0:1] == 'w' or mode[0:1] == 'a':
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-        else:
-            raise IOError, "Mode " + mode + " not supported"
-
-        self.fileobj = fileobj
-        self.offset = 0
-        self.mtime = mtime
-
-        if self.mode == WRITE:
-            self._write_gzip_header()
-
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
-    def __repr__(self):
-        s = repr(self.fileobj)
-        return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
-
-    def _check_closed(self):
-        """Raises a ValueError if the underlying file object has been closed.
-
-        """
-        if self.closed:
-            raise ValueError('I/O operation on closed file.')
-
-    def _init_write(self, filename):
-        self.name = filename
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-        self.writebuf = []
-        self.bufsize = 0
-
-    def _write_gzip_header(self):
-        self.fileobj.write('\037\213')             # magic header
-        self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
-        flags = 0
-        if fname:
-            flags = FNAME
-        self.fileobj.write(chr(flags))
-        mtime = self.mtime
-        if mtime is None:
-            mtime = time.time()
-        write32u(self.fileobj, long(mtime))
-        self.fileobj.write('\002')
-        self.fileobj.write('\377')
-        if fname:
-            self.fileobj.write(fname + '\000')
-
-    def _init_read(self):
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-
-    def _read_gzip_header(self):
-        magic = self.fileobj.read(2)
-        if magic != '\037\213':
-            raise IOError, 'Not a gzipped file'
-        method = ord( self.fileobj.read(1) )
-        if method != 8:
-            raise IOError, 'Unknown compression method'
-        flag = ord( self.fileobj.read(1) )
-        self.mtime = read32(self.fileobj)
-        # extraflag = self.fileobj.read(1)
-        # os = self.fileobj.read(1)
-        self.fileobj.read(2)
-
-        if flag & FEXTRA:
-            # Read & discard the extra field, if present
-            xlen = ord(self.fileobj.read(1))
-            xlen = xlen + 256*ord(self.fileobj.read(1))
-            self.fileobj.read(xlen)
-        if flag & FNAME:
-            # Read and discard a null-terminated string containing the filename
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FCOMMENT:
-            # Read and discard a null-terminated string containing a comment
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FHCRC:
-            self.fileobj.read(2)     # Read & discard the 16-bit header CRC
-
-    def write(self,data):
-        self._check_closed()
-        if self.mode != WRITE:
-            import errno
-            raise IOError(errno.EBADF, "write() on read-only GzipFile object")
-
-        if self.fileobj is None:
-            raise ValueError, "write() on closed GzipFile object"
-
-        # Convert data type if called by io.BufferedWriter.
-        if isinstance(data, memoryview):
-            data = data.tobytes()
-
-        if len(data) > 0:
-            self.size = self.size + len(data)
-            self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
-            self.offset += len(data)
-
-        return len(data)
-
-    def read(self, size=-1):
-        self._check_closed()
-        if self.mode != READ:
-            import errno
-            raise IOError(errno.EBADF, "read() on write-only GzipFile object")
-
-        if self.extrasize <= 0 and self.fileobj is None:
-            return ''
-
-        readsize = 1024
-        if size < 0:        # get the whole thing
-            try:
-                while True:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                size = self.extrasize
-        elif size == 0:
-            return ""
-        else:               # just get some more of it
-            try:
-                while size > self.extrasize:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                if size > self.extrasize:
-                    size = self.extrasize
-
-        offset = self.offset - self.extrastart
-        chunk = self.extrabuf[offset: offset + size]
-        self.extrasize = self.extrasize - size
-
-        self.offset += size
-        return chunk
-
-    def _unread(self, buf):
-        self.extrasize = len(buf) + self.extrasize
-        self.offset -= len(buf)
-
-    def _read(self, size=1024):
-        if self.fileobj is None:
-            raise EOFError, "Reached EOF"
-
-        if self._new_member:
-            # If the _new_member flag is set, we have to
-            # jump to the next member, if there is one.
-            #
-            # First, check if we're at the end of the file;
-            # if so, it's time to stop; no more members to read.
-            pos = self.fileobj.tell()   # Save current position
-            self.fileobj.seek(0, 2)     # Seek to end of file
-            if pos == self.fileobj.tell():
-                raise EOFError, "Reached EOF"
-            else:
-                self.fileobj.seek( pos ) # Return to original position
-
-            self._init_read()
-            self._read_gzip_header()
-            self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
-            self._new_member = False
-
-        # Read a chunk of data from the file
-        buf = self.fileobj.read(size)
-
-        # If the EOF has been reached, flush the decompression object
-        # and mark this object as finished.
-
-        if buf == "":
-            uncompress = self.decompress.flush()
-            self._read_eof()
-            self._add_read_data( uncompress )
-            raise EOFError, 'Reached EOF'
-
-        uncompress = self.decompress.decompress(buf)
-        self._add_read_data( uncompress )
-
-        if self.decompress.unused_data != "":
-            # Ending case: we've come to the end of a member in the file,
-            # so seek back to the start of the unused data, finish up
-            # this member, and read a new gzip header.
-            # (The number of bytes to seek back is the length of the unused
-            # data, minus 8 because _read_eof() will rewind a further 8 bytes)
-            self.fileobj.seek( -len(self.decompress.unused_data)+8, 1)
-
-            # Check the CRC and file size, and set the flag so we read
-            # a new member on the next call
-            self._read_eof()
-            self._new_member = True
-
-    def _add_read_data(self, data):
-        self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-        offset = self.offset - self.extrastart
-        self.extrabuf = self.extrabuf[offset:] + data
-        self.extrasize = self.extrasize + len(data)
-        self.extrastart = self.offset
-        self.size = self.size + len(data)
-
-    def _read_eof(self):
-        # We've read to the end of the file, so we have to rewind in order
-        # to reread the 8 bytes containing the CRC and the file size.
-        # We check the that the computed CRC and size of the
-        # uncompressed data matches the stored values.  Note that the size
-        # stored is the true file size mod 2**32.
-        self.fileobj.seek(-8, 1)
-        crc32 = read32(self.fileobj)
-        isize = read32(self.fileobj)  # may exceed 2GB
-        if crc32 != self.crc:
-            raise IOError("CRC check failed %s != %s" % (hex(crc32),
-                                                         hex(self.crc)))
-        elif isize != (self.size & 0xffffffffL):
-            raise IOError, "Incorrect length of data produced"
-
-        # Gzip files can be padded with zeroes and still have archives.
-        # Consume all zero bytes and set the file position to the first
-        # non-zero byte. See http://www.gzip.org/#faq8
-        c = "\x00"
-        while c == "\x00":
-            c = self.fileobj.read(1)
-        if c:
-            self.fileobj.seek(-1, 1)
-
-    @property
-    def closed(self):
-        return self.fileobj is None
-
-    def close(self):
-        if self.fileobj is None:
-            return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
-
-    def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
-        self._check_closed()
-        if self.mode == WRITE:
-            # Ensure the compressor's buffer is flushed
-            self.fileobj.write(self.compress.flush(zlib_mode))
-            self.fileobj.flush()
-
-    def fileno(self):
-        """Invoke the underlying file object's fileno() method.
-
-        This will raise AttributeError if the underlying file object
-        doesn't support fileno().
-        """
-        return self.fileobj.fileno()
-
-    def rewind(self):
-        '''Return the uncompressed stream file position indicator to the
-        beginning of the file'''
-        if self.mode != READ:
-            raise IOError("Can't rewind in write mode")
-        self.fileobj.seek(0)
-        self._new_member = True
-        self.extrabuf = ""
-        self.extrasize = 0
-        self.extrastart = 0
-        self.offset = 0
-
-    def readable(self):
-        return self.mode == READ
-
-    def writable(self):
-        return self.mode == WRITE
-
-    def seekable(self):
-        return True
-
-    def seek(self, offset, whence=0):
-        if whence:
-            if whence == 1:
-                offset = self.offset + offset
-            else:
-                raise ValueError('Seek from end not supported')
-        if self.mode == WRITE:
-            if offset < self.offset:
-                raise IOError('Negative seek in write mode')
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.write(1024 * '\0')
-            self.write((count % 1024) * '\0')
-        elif self.mode == READ:
-            if offset == self.offset:
-                self.read(0) # to make sure that this file is open
-                return self.offset
-            if offset < self.offset:
-                # for negative seek, rewind and do positive seek
-                self.rewind()
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.read(1024)
-            self.read(count % 1024)
-
-        return self.offset
-
-    def readline(self, size=-1):
-        if size < 0:
-            # Shortcut common case - newline found in buffer.
-            offset = self.offset - self.extrastart
-            i = self.extrabuf.find('\n', offset) + 1
-            if i > 0:
-                self.extrasize -= i - offset
-                self.offset += i - offset
-                return self.extrabuf[offset: i]
-
-            size = sys.maxint
-            readsize = self.min_readsize
-        else:
-            readsize = size
-        bufs = []
-        while size != 0:
-            c = self.read(readsize)
-            i = c.find('\n')
-
-            # We set i=size to break out of the loop under two
-            # conditions: 1) there's no newline, and the chunk is
-            # larger than size, or 2) there is a newline, but the
-            # resulting line would be longer than 'size'.
-            if (size <= i) or (i == -1 and len(c) > size):
-                i = size - 1
-
-            if i >= 0 or c == '':
-                bufs.append(c[:i + 1])    # Add portion of last chunk
-                self._unread(c[i + 1:])   # Push back rest of chunk
-                break
-
-            # Append chunk to list, decrease 'size',
-            bufs.append(c)
-            size = size - len(c)
-            readsize = min(size, readsize * 2)
-        if readsize > self.min_readsize:
-            self.min_readsize = min(readsize, self.min_readsize * 2, 512)
-        return ''.join(bufs) # Return resulting line
-
-
-def _test():
-    # Act like gzip; with -d, act like gunzip.
-    # The input file is not deleted, however, nor are any other gzip
-    # options or features supported.
-    args = sys.argv[1:]
-    decompress = args and args[0] == "-d"
-    if decompress:
-        args = args[1:]
-    if not args:
-        args = ["-"]
-    for arg in args:
-        if decompress:
-            if arg == "-":
-                f = GzipFile(filename="", mode="rb", fileobj=sys.stdin)
-                g = sys.stdout
-            else:
-                if arg[-3:] != ".gz":
-                    print "filename doesn't end in .gz:", repr(arg)
-                    continue
-                f = open(arg, "rb")
-                g = __builtin__.open(arg[:-3], "wb")
-        else:
-            if arg == "-":
-                f = sys.stdin
-                g = GzipFile(filename="", mode="wb", fileobj=sys.stdout)
-            else:
-                f = __builtin__.open(arg, "rb")
-                g = open(arg + ".gz", "wb")
-        while True:
-            chunk = f.read(1024)
-            if not chunk:
-                break
-            g.write(chunk)
-        if g is not sys.stdout:
-            g.close()
-        if f is not sys.stdin:
-            f.close()
-
-if __name__ == '__main__':
-    _test()

lib-python/modified-2.7/tarfile.py

        the high bit set. So we calculate two checksums, unsigned and
        signed.
     """
-    unsigned_chksum = 256 + sum(struct.unpack("148B8x356B", buf[:512]))
-    signed_chksum = 256 + sum(struct.unpack("148b8x356b", buf[:512]))
+    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
     return unsigned_chksum, signed_chksum
 
 def copyfileobj(src, dst, length=None):
     if length is None:
         shutil.copyfileobj(src, dst)
         return
+
     BUFSIZE = 16 * 1024
     blocks, remainder = divmod(length, BUFSIZE)
     for b in xrange(blocks):
         if self.closed:
             raise ValueError("I/O operation on closed file")
 
+        buf = ""
         if self.buffer:
             if size is None:
-                buf = self.buffer + self.fileobj.read()
+                buf = self.buffer
                 self.buffer = ""
             else:
                 buf = self.buffer[:size]
                 self.buffer = self.buffer[size:]
-                buf += self.fileobj.read(size - len(buf))
+
+        if size is None:
+            buf += self.fileobj.read()
         else:
-            if size is None:
-                buf = self.fileobj.read()
-            else:
-                buf = self.fileobj.read(size)
+            buf += self.fileobj.read(size - len(buf))
 
         self.position += len(buf)
         return buf

lib-python/modified-2.7/test/test_multiprocessing.py

 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50

lib-python/modified-2.7/test/test_sys_settrace.py

         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 

lib_pypy/_functools.py

 
     def __call__(self, *fargs, **fkeywords):
         if self.keywords is not None:
-            fkeywords.update(self.keywords)
+            fkeywords = dict(self.keywords, **fkeywords)
         return self.func(*(self.args + fargs), **fkeywords)

lib_pypy/greenlet.py

 
     @property
     def gr_frame(self):
-        raise NotImplementedError("attribute 'gr_frame' of greenlet objects")
+        # xxx this doesn't work when called on either the current or
+        # the main greenlet of another thread
+        if self is getcurrent():
+            return None
+        if self.__main:
+            self = getcurrent()
+        f = _continulet.__reduce__(self)[2][0]
+        if not f:
+            return None
+        return f.f_back.f_back.f_back   # go past start(), __switch(), switch()
 
 # ____________________________________________________________
 # Internal stuff

lib_pypy/pypy_test/test_stackless_pickling.py

-"""
-this test should probably not run from CPython or py.py.
-I'm not entirely sure, how to do that.
-"""
 from __future__ import absolute_import
 from py.test import skip
 try:
 
 class Test_StacklessPickling:
 
+    def test_pickle_main_coroutine(self):
+        import stackless, pickle
+        s = pickle.dumps(stackless.coroutine.getcurrent())
+        print s
+        c = pickle.loads(s)
+        assert c is stackless.coroutine.getcurrent()
+
     def test_basic_tasklet_pickling(self):
-        try:
-            import stackless
-        except ImportError:
-            skip("can't load stackless and don't know why!!!")
+        import stackless
         from stackless import run, schedule, tasklet
         import pickle
 

lib_pypy/pyrepl/completing_reader.py

 
     def after_command(self, cmd):
         super(CompletingReader, self).after_command(cmd)
-        if not isinstance(cmd, complete) and not isinstance(cmd, self_insert):
+        if not isinstance(cmd, self.commands['complete']) \
+           and not isinstance(cmd, self.commands['self_insert']):
             self.cmpltn_reset()
 
     def calc_screen(self):

lib_pypy/stackless.py

 """
 
 
-import traceback
 import _continuation
 
 class TaskletExit(Exception):
 CoroutineExit = TaskletExit
 
 
+def _coroutine_getcurrent():
+    "Returns the current coroutine (i.e. the one which called this function)."
+    try:
+        return _tls.current_coroutine
+    except AttributeError:
+        # first call in this thread: current == main
+        return _coroutine_getmain()
+
+def _coroutine_getmain():
+    try:
+        return _tls.main_coroutine
+    except AttributeError:
+        # create the main coroutine for this thread
+        continulet = _continuation.continulet
+        main = coroutine()
+        main._frame = continulet.__new__(continulet)
+        main._is_started = -1
+        _tls.current_coroutine = _tls.main_coroutine = main
+        return _tls.main_coroutine
+
+
 class coroutine(object):
-    "we can't have continulet as a base, because continulets can't be rebound"
+    _is_started = 0      # 0=no, 1=yes, -1=main
 
     def __init__(self):
         self._frame = None
-        self.is_zombie = False
-
-    def __getattr__(self, attr):
-        return getattr(self._frame, attr)
-
-    def __del__(self):
-        self.is_zombie = True
-        del self._frame
-        self._frame = None
 
     def bind(self, func, *argl, **argd):
         """coro.bind(f, *argl, **argd) -> None.
            binds function f to coro. f will be called with
            arguments *argl, **argd
         """
-        if self._frame is None or not self._frame.is_pending():
-            def run(c):
-                _tls.current_coroutine = self
-                return func(*argl, **argd)
-            self._frame = frame = _continuation.continulet(run)
-        else:
+        if self.is_alive:
             raise ValueError("cannot bind a bound coroutine")
+        def run(c):
+            _tls.current_coroutine = self
+            self._is_started = 1
+            return func(*argl, **argd)
+        self._is_started = 0
+        self._frame = _continuation.continulet(run)
 
     def switch(self):
         """coro.switch() -> returnvalue
            f finishes, the returnvalue is that of f, otherwise
            None is returned
         """
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.switch(to=self._frame)
         finally:
 
     def kill(self):
         """coro.kill() : kill coroutine coro"""
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.throw(CoroutineExit, to=self._frame)
         finally:
             _tls.current_coroutine = current
 
-    def _is_alive(self):
-        if self._frame is None:
-            return False
-        return not self._frame.is_pending()
-    is_alive = property(_is_alive)
-    del _is_alive
+    @property
+    def is_alive(self):
+        return self._is_started < 0 or (
+            self._frame is not None and self._frame.is_pending())
 
-    def getcurrent():
-        """coroutine.getcurrent() -> the currently running coroutine"""
-        return _getcurrent()
-    getcurrent = staticmethod(getcurrent)
+    @property
+    def is_zombie(self):
+        return self._is_started > 0 and not self._frame.is_pending()
+
+    getcurrent = staticmethod(_coroutine_getcurrent)
 
     def __reduce__(self):
-        raise TypeError, 'pickling is not possible based upon continulets'
+        if self._is_started < 0:
+            return _coroutine_getmain, ()
+        else:
+            return type(self), (), self.__dict__
 
 
-def _getcurrent():
-    "Returns the current coroutine (i.e. the one which called this function)."
-    try:
-        return _tls.current_coroutine
-    except AttributeError:
-        # first call in this thread: current == main
-        _coroutine_create_main()
-        return _tls.current_coroutine
-
 try:
     from thread import _local
 except ImportError:
 
 _tls = _local()
 
-def _coroutine_create_main():
-    # create the main coroutine for this thread
-    _tls.current_coroutine = None
-    main_coroutine = coroutine()
-    typ = _continuation.continulet
-    main_coroutine._frame = typ.__new__(typ)
-    _tls.main_coroutine = main_coroutine
-    _tls.current_coroutine = main_coroutine
+
+# ____________________________________________________________
 
 
 from collections import deque
     _last_task = next
     assert not next.blocked
     if next is not current:
-        #try:
-            next.switch()
-        #except CoroutineExit:  --- they are the same anyway
-        #    raise TaskletExit
+        next.switch()
     return current
 
 def set_schedule_callback(callback):
         raise self.type, self.value, self.traceback
 
 #
-# helpers for pickling
-#
-
-_stackless_primitive_registry = {}
-
-def register_stackless_primitive(thang, retval_expr='None'):
-    import types
-    func = thang
-    if isinstance(thang, types.MethodType):
-        func = thang.im_func
-    code = func.func_code
-    _stackless_primitive_registry[code] = retval_expr
-    # It is not too nice to attach info via the code object, but
-    # I can't think of a better solution without a real transform.
-
-def rewrite_stackless_primitive(coro_state, alive, tempval):
-    flags, frame, thunk, parent = coro_state
-    while frame is not None:
-        retval_expr = _stackless_primitive_registry.get(frame.f_code)
-        if retval_expr:
-            # this tasklet needs to stop pickling here and return its value.
-            tempval = eval(retval_expr, globals(), frame.f_locals)
-            coro_state = flags, frame, thunk, parent
-            break
-        frame = frame.f_back
-    return coro_state, alive, tempval
-
-#
 #
 
 class channel(object):
         """
         return self._channel_action(None, -1)
 
-    register_stackless_primitive(receive, retval_expr='receiver.tempval')
-
     def send_exception(self, exp_type, msg):
         self.send(bomb(exp_type, exp_type(msg)))
 
         the runnables list.
         """
         return self._channel_action(msg, 1)
-            
-    register_stackless_primitive(send)
-            
+
+
 class tasklet(coroutine):
     """
     A tasklet object represents a tiny task in a Python thread.
 
         self.func = None
         coroutine.bind(self, _func)
-        back = _getcurrent()
+        back = _coroutine_getcurrent()
         coroutine.switch(self)
         self.alive = True
         _scheduler_append(self)
             raise RuntimeError, "The current tasklet cannot be removed."
             # not sure if I will revive this  " Use t=tasklet().capture()"
         _scheduler_remove(self)
-        
-    def __reduce__(self):
-        one, two, coro_state = coroutine.__reduce__(self)
-        assert one is coroutine
-        assert two == ()
-        # we want to get rid of the parent thing.
-        # for now, we just drop it
-        a, frame, c, d = coro_state
-
-        # Removing all frames related to stackless.py.
-        # They point to stuff we don't want to be pickled.
-
-        pickleframe = frame
-        while frame is not None:
-            if frame.f_code == schedule.func_code:
-                # Removing everything including and after the
-                # call to stackless.schedule()
-                pickleframe = frame.f_back
-                break
-            frame = frame.f_back
-        if d:
-            assert isinstance(d, coroutine)
-        coro_state = a, pickleframe, c, None
-        coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval)
-        inst_dict = self.__dict__.copy()
-        inst_dict.pop('tempval', None)
-        return self.__class__, (), (coro_state, alive, tempval, inst_dict)
-
-    def __setstate__(self, (coro_state, alive, tempval, inst_dict)):
-        coroutine.__setstate__(self, coro_state)
-        self.__dict__.update(inst_dict)
-        self.alive = alive
-        self.tempval = tempval
 
 def getmain():
     """
     global _last_task
     _global_task_id = 0
     _main_tasklet = coroutine.getcurrent()
-    try:
-        _main_tasklet.__class__ = tasklet
-    except TypeError: # we are running pypy-c
-        class TaskletProxy(object):
-            """TaskletProxy is needed to give the _main_coroutine tasklet behaviour"""
-            def __init__(self, coro):
-                self._coro = coro
-
-            def __getattr__(self,attr):
-                return getattr(self._coro,attr)
-
-            def __str__(self):
-                return '<tasklet %s a:%s>' % (self._task_id, self.is_alive)
-
-            def __reduce__(self):
-                return getmain, ()
-
-            __repr__ = __str__
-
-
-        global _main_coroutine
-        _main_coroutine = _main_tasklet
-        _main_tasklet = TaskletProxy(_main_tasklet)
-        assert _main_tasklet.is_alive and not _main_tasklet.is_zombie
+    _main_tasklet.__class__ = tasklet         # XXX HAAAAAAAAAAAAAAAAAAAAACK
     _last_task = _main_tasklet
     tasklet._init.im_func(_main_tasklet, label='main')
     _squeue = deque()

py/_code/source.py

                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, end
+        return start, len(self)
 
     def getblockend(self, lineno):
         # XXX

pypy/annotation/policy.py

 # base annotation policy for specialization
 from pypy.annotation.specialize import default_specialize as default
-from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype
+from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
 from pypy.annotation.specialize import memo, specialize_call_location
 # for some reason, model must be imported first,
 # or we create a cycle.
     default_specialize = staticmethod(default)
     specialize__memo = staticmethod(memo)
     specialize__arg = staticmethod(specialize_argvalue) # specialize:arg(N)
+    specialize__arg_or_var = staticmethod(specialize_arg_or_var)
     specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N)
     specialize__arglistitemtype = staticmethod(specialize_arglistitemtype)
     specialize__call_location = staticmethod(specialize_call_location)

pypy/annotation/specialize.py

     key = tuple(key)
     return maybe_star_args(funcdesc, key, args_s)
 
+def specialize_arg_or_var(funcdesc, args_s, *argindices):
+    for argno in argindices:
+        if not args_s[argno].is_constant():
+            break
+    else:
+        # all constant
+        return specialize_argvalue(funcdesc, args_s, *argindices)
+    # some not constant
+    return maybe_star_args(funcdesc, None, args_s)
+
 def specialize_argtype(funcdesc, args_s, *argindices):
     key = tuple([args_s[i].knowntype for i in argindices])
     for cls in key:

pypy/annotation/test/test_annrpython.py

         assert len(executedesc._cache[(0, 'star', 2)].startblock.inputargs) == 4
         assert len(executedesc._cache[(1, 'star', 3)].startblock.inputargs) == 5
 
+    def test_specialize_arg_or_var(self):
+        def f(a):
+            return 1
+        f._annspecialcase_ = 'specialize:arg_or_var(0)'
+
+        def fn(a):
+            return f(3) + f(a)
+
+        a = self.RPythonAnnotator()
+        a.build_types(fn, [int])
+        executedesc = a.bookkeeper.getdesc(f)
+        assert sorted(executedesc._cache.keys()) == [None, (3,)]
+        # we got two different special
+
     def test_specialize_call_location(self):
         def g(a):
             return a
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():

pypy/annotation/unaryop.py

         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 

pypy/doc/project-ideas.rst

 this is an ideal task to get started, because it does not require any deep
 knowledge of the internals.
 
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
 Translation Toolchain
 ---------------------
 

pypy/interpreter/baseobjspace.py

 from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag
 from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.error import new_exception_class
+from pypy.interpreter.error import new_exception_class, typed_unwrap_error_msg
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals
 from pypy.tool.cache import Cache
 from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated, newlist
+from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.timer import DummyTimer, Timer
 from pypy.rlib.rarithmetic import r_uint
     def _set_mapdict_storage_and_map(self, storage, map):
         raise NotImplementedError
 
+    # -------------------------------------------------------------------
+
+    def str_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "string", self)
+        raise OperationError(space.w_TypeError, w_msg)
+
+    def unicode_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "unicode", self))
+
+    def int_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def uint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def bigint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+
 
 class Wrappable(W_Root):
     """A subclass of Wrappable is an internal, interpreter-level class
         ec.c_call_trace(frame, w_func, args)
         try:
             w_res = self.call_args(w_func, args)
-        except OperationError, e:
+        except OperationError:
             ec.c_exception_trace(frame, w_func)
             raise
         ec.c_return_trace(frame, w_func, args)
     def isinstance_w(self, w_obj, w_type):
         return self.is_true(self.isinstance(w_obj, w_type))
 
+    def id(self, w_obj):
+        return self.wrap(compute_unique_id(w_obj))
+
     # The code below only works
     # for the simple case (new-style instance).
     # These methods are patched with the full logic by the __builtin__
 
     def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
         "NOT_RPYTHON: For internal debugging."
-        import types
-        from pypy.interpreter.pycode import PyCode
         if isinstance(expression, str):
             compiler = self.createcompiler()
             expression = compiler.compile(expression, '?', 'eval', 0,
     def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
               filename=None):
         "NOT_RPYTHON: For internal debugging."
-        import types
         if filename is None:
             filename = '?'
         from pypy.interpreter.pycode import PyCode
             return None
         return self.str_w(w_obj)
 
+    def str_w(self, w_obj):
+        return w_obj.str_w(self)
+
+    def int_w(self, w_obj):
+        return w_obj.int_w(self)
+
+    def uint_w(self, w_obj):
+        return w_obj.uint_w(self)
+
+    def bigint_w(self, w_obj):
+        return w_obj.bigint_w(self)
+
     def realstr_w(self, w_obj):
         # Like str_w, but only works if w_obj is really of type 'str'.
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
                                  self.wrap('argument must be a string'))
         return self.str_w(w_obj)
 
+    def unicode_w(self, w_obj):
+        return w_obj.unicode_w(self)
+
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.

pypy/interpreter/error.py

     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def typed_unwrap_error_msg(space, expected, w_obj):
+    type_name = space.type(w_obj).getname(space)
+    return space.wrap("expected %s, got %s object" % (expected, type_name))

pypy/interpreter/executioncontext.py

             self.w_tracefunc = w_func
             self.space.frame_trace_action.fire()
 
+    def gettrace(self):
+        return self.w_tracefunc
+
     def setprofile(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
         elif interval > MAX:
             interval = MAX
         self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
 
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)

pypy/interpreter/pyframe.py

         make_sure_not_resized(self.locals_stack_w)
         check_nonneg(self.nlocals)
         #
-        if space.config.objspace.honor__builtins__ and w_globals is not None:
+        if space.config.objspace.honor__builtins__:
             self.builtin = space.builtin.pick_builtin(w_globals)
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
         # class bodies only have CO_NEWLOCALS.

pypy/interpreter/test/test_executioncontext.py

         assert i == 9
 
     def test_periodic_action(self):
+        from pypy.interpreter.executioncontext import ActionFlag
 
         class DemoAction(executioncontext.PeriodicAsyncAction):
             counter = 0
 
         space = self.space
         a2 = DemoAction(space)
-        space.actionflag.register_periodic_action(a2, True)
         try:
-            for i in range(500):
-                space.appexec([], """():
-                    n = 5
-                    return n + 2
-                """)
-        except Finished:
-            pass
-        checkinterval = space.actionflag.getcheckinterval()
-        assert checkinterval / 10 < i < checkinterval * 1.1
+            space.actionflag.setcheckinterval(100)
+            space.actionflag.register_periodic_action(a2, True)
+            try:
+                for i in range(500):
+                    space.appexec([], """():
+                        n = 5
+                        return n + 2
+                    """)
+            except Finished:
+                pass
+        finally:
+            space.actionflag = ActionFlag()   # reset to default
+        assert 10 < i < 110
 
     def test_llprofile(self):
         l = []

pypy/jit/codewriter/effectinfo.py

     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 

pypy/jit/codewriter/jtransform.py

     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]

pypy/jit/codewriter/support.py

 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support

pypy/jit/codewriter/test/test_flatten.py

                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"

pypy/jit/codewriter/test/test_longlong.py

             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,

pypy/jit/metainterp/optimizeopt/__init__.py

 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]

pypy/jit/metainterp/optimizeopt/earlyforce.py

+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    

pypy/jit/metainterp/optimizeopt/fficall.py

             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()

pypy/jit/metainterp/optimizeopt/heap.py

             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],

pypy/jit/metainterp/optimizeopt/optimizer.py

 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):