Commits

Maciej Fijalkowski committed 9cb706d Merge

merge default

  • Participants
  • Parent commits 9446de5, 820183b
  • Branches kill-gen-store-back-in

Comments (0)

Files changed (283)

 ^rpython/translator/cli/src/pypylib\.dll$
 ^rpython/translator/cli/src/query\.exe$
 ^rpython/translator/cli/src/main\.exe$
+^lib_pypy/__pycache__$
 ^lib_pypy/ctypes_config_cache/_.+_cache\.py$
 ^lib_pypy/ctypes_config_cache/_.+_.+_\.py$
 ^rpython/translator/cli/query-descriptions$
+
+all: pypy-c
+
+pypy-c:
+	@echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM"
+	@sleep 3
+	rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
 
     http://pypy.org/
 
-The getting-started document will help guide you:
+If you want to help developing PyPy, this document might help you:
 
-    http://doc.pypy.org/en/latest/getting-started.html
+    http://doc.pypy.org/
 
 It will also point you to the rest of the documentation which is generated
 from files in the pypy/doc directory within the source repositories. Enjoy
 and send us feedback!
 
     the pypy-dev team <pypy-dev@python.org>
+
+Building
+========
+
+build with::
+
+  rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+
+This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
+to use virtualenv with the resulting pypy-c as the interpreter, you can
+find more details about various installation schemes here:
+
+http://doc.pypy.org/en/latest/getting-started.html#installing-pypy

lib-python/2.7/json/decoder.py

         if nextchar == '}':
             if object_pairs_hook is not None:
                 result = object_pairs_hook(pairs)
-                return result, end
+                return result, end + 1
             pairs = {}
             if object_hook is not None:
                 pairs = object_hook(pairs)

lib-python/2.7/json/tests/test_decode.py

                                     object_pairs_hook=OrderedDict,
                                     object_hook=lambda x: None),
                          OrderedDict(p))
+        self.assertEqual(self.loads("{}", object_pairs_hook=list), [])
 
 
 class TestPyDecode(TestDecode, PyTest): pass

lib-python/2.7/test/test_descr.py

         list.__init__(a, sequence=[0, 1, 2])
         self.assertEqual(a, [0, 1, 2])
 
+    @unittest.skipIf(test_support.check_impl_detail(pypy=True) and
+                     sys.platform == 'win32',
+                     "XXX: https://bugs.pypy.org/issue1461")
     def test_recursive_call(self):
         # Testing recursive __call__() by setting to instance of class...
         class A(object):

lib-python/conftest.py

 """
 
 test configuration(s) for running CPython's regression
-test suite on top of PyPy 
+test suite on top of PyPy
 
 """
 import py
 import sys
 import pypy
 import re
-from pypy.interpreter.gateway import ApplevelClass 
+from pypy.interpreter.gateway import ApplevelClass
 from pypy.interpreter.error import OperationError
-from pypy.interpreter.module import Module as PyPyModule 
+from pypy.interpreter.module import Module as PyPyModule
 from pypy.interpreter.main import run_string, run_file
 
-# the following adds command line options as a side effect! 
-from pypy.conftest import option as pypy_option 
+# the following adds command line options as a side effect!
+from pypy.conftest import option as pypy_option
 
-from pypy.tool.pytest import appsupport 
+from pypy.tool.pytest import appsupport
 from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir
 from rpython.config.parse import parse_info
 
 pytest_plugins = "resultlog",
 rsyncdirs = ['.', '../pypy/']
 
-# 
-# Interfacing/Integrating with py.test's collection process 
+#
+# Interfacing/Integrating with py.test's collection process
 #
 
 def pytest_addoption(parser):
-    group = parser.getgroup("complicance testing options") 
-    group.addoption('-T', '--timeout', action="store", type="string", 
-       default="1000", dest="timeout", 
-       help="fail a test module after the given timeout. "
-            "specify in seconds or 'NUMmp' aka Mega-Pystones")
-    group.addoption('--pypy', action="store", type="string",
-       dest="pypy",  help="use given pypy executable to run lib-python tests. "
-                          "This will run the tests directly (i.e. not through py.py)")
+    group = parser.getgroup("complicance testing options")
+    group.addoption('-T', '--timeout', action="store", type="string",
+                    default="1000", dest="timeout",
+                    help="fail a test module after the given timeout. "
+                         "specify in seconds or 'NUMmp' aka Mega-Pystones")
+    group.addoption('--pypy', action="store", type="string", dest="pypy",
+                    help="use given pypy executable to run lib-python tests. "
+                         "This will run the tests directly (i.e. not through py.py)")
     group.addoption('--filter', action="store", type="string", default=None,
-                    dest="unittest_filter",  help="Similar to -k, XXX")
+                    dest="unittest_filter", help="Similar to -k, XXX")
 
-def gettimeout(timeout): 
+def gettimeout(timeout):
     from rpython.translator.test import rpystone
-    if timeout.endswith('mp'): 
+    if timeout.endswith('mp'):
         megapystone = float(timeout[:-2])
         t, stone = pystone.Proc0(10000)
-        pystonetime = t/stone 
-        seconds = megapystone  * 1000000 * pystonetime
-        return seconds 
-    return float(timeout) 
+        pystonetime = t/stone
+        seconds = megapystone * 1000000 * pystonetime
+        return seconds
+    return float(timeout)
 
 # ________________________________________________________________________
 #
-# classification of all tests files (this is ongoing work) 
+# classification of all tests files (this is ongoing work)
 #
 
-class RegrTest: 
-    """ Regression Test Declaration.""" 
-    def __init__(self, basename, core=False,
-                                 compiler=None, 
-                                 usemodules = '',
-                                 skip=None): 
-        self.basename = basename 
+class RegrTest:
+    """ Regression Test Declaration."""
+    def __init__(self, basename, core=False, compiler=None, usemodules='',
+                 skip=None):
+        self.basename = basename
         self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
-        self._compiler = compiler 
+        self._compiler = compiler
         self.core = core
         self.skip = skip
         assert self.getfspath().check(), "%r not found!" % (basename,)
 
     def usemodules(self):
-        return self._usemodules #+ pypy_option.usemodules
+        return self._usemodules  # + pypy_option.usemodules
     usemodules = property(usemodules)
 
-    def compiler(self): 
-        return self._compiler #or pypy_option.compiler 
+    def compiler(self):
+        return self._compiler  # or pypy_option.compiler
     compiler = property(compiler)
 
-    def ismodified(self): 
+    def ismodified(self):
         #XXX: ask hg
         return None
 
-    def getfspath(self): 
+    def getfspath(self):
         return testdir.join(self.basename)
 
-    def run_file(self, space): 
+    def run_file(self, space):
         fspath = self.getfspath()
         assert fspath.check()
-        modname = fspath.purebasename 
+        modname = fspath.purebasename
         space.appexec([], '''():
             from test import %(modname)s
             m = %(modname)s
     RegrTest('test_textwrap.py'),
     RegrTest('test_thread.py', usemodules="thread", core=True),
     RegrTest('test_threaded_import.py', usemodules="thread", core=True),
-    RegrTest('test_threadedtempfile.py', 
+    RegrTest('test_threadedtempfile.py',
              usemodules="thread", core=False),
     RegrTest('test_threading.py', usemodules="thread", core=True),
     RegrTest('test_threading_local.py', usemodules="thread", core=True),
 
 def pytest_configure(config):
     config._basename2spec = cache = {}
-    for x in testmap: 
+    for x in testmap:
         cache[x.basename] = x
 
 def pytest_collect_file(path, parent, __multicall__):
     return RunFileExternal(path.basename, parent=parent, regrtest=regrtest)
 
 class RunFileExternal(py.test.collect.File):
-    def __init__(self, name, parent, regrtest): 
-        super(RunFileExternal, self).__init__(name, parent) 
-        self.regrtest = regrtest 
+    def __init__(self, name, parent, regrtest):
+        super(RunFileExternal, self).__init__(name, parent)
+        self.regrtest = regrtest
         self.fspath = regrtest.getfspath()
 
-    def collect(self): 
-        if self.regrtest.ismodified(): 
+    def collect(self):
+        if self.regrtest.ismodified():
             name = 'modified'
         else:
             name = 'unmodified'
-        return [ReallyRunFileExternal(name, parent=self)] 
+        return [ReallyRunFileExternal(name, parent=self)]
 
 #
-# testmethod: 
+# testmethod:
 # invoking in a separate process: py.py TESTFILE
 #
 import os
 import time
 import getpass
 
-class ReallyRunFileExternal(py.test.collect.Item): 
+class ReallyRunFileExternal(py.test.collect.Item):
     class ExternalFailure(Exception):
         """Failure in running subprocess"""
 
-    def getinvocation(self, regrtest): 
-        fspath = regrtest.getfspath() 
-        python = sys.executable 
+    def getinvocation(self, regrtest):
+        fspath = regrtest.getfspath()
+        python = sys.executable
         pypy_script = pypydir.join('bin', 'pyinteractive.py')
         alarm_script = pypydir.join('tool', 'alarm.py')
         if sys.platform == 'win32':
             watchdog_name = 'watchdog.py'
         watchdog_script = rpythondir.join('tool', watchdog_name)
 
-        regr_script = pypydir.join('tool', 'pytest', 
+        regr_script = pypydir.join('tool', 'pytest',
                                    'run-script', 'regrverbose.py')
-        
+
         regrrun = str(regr_script)
         option = self.config.option
         TIMEOUT = gettimeout(option.timeout.lower())
             if not execpath.check():
                 execpath = py.path.local.sysfind(option.pypy)
             if not execpath:
-                raise LookupError("could not find executable %r" %
-                                  (option.pypy,))
+                raise LookupError("could not find executable %r" % option.pypy)
 
             # check modules
             info = py.process.cmdexec("%s --info" % execpath)
                 if info.get('objspace.usemodules.%s' % mod) is not True:
                     py.test.skip("%s module not included in %s" % (mod,
                                                                    execpath))
-                    
-            cmd = "%s %s %s" %(
-                execpath, 
-                regrrun, fspath.purebasename)
 
+            cmd = "%s %s %s" % (execpath, regrrun, fspath.purebasename)
             # add watchdog for timing out
-            cmd = "%s %s %s %s" %(
-                python, watchdog_script, TIMEOUT,
-                cmd)
+            cmd = "%s %s %s %s" % (python, watchdog_script, TIMEOUT, cmd)
         else:
             pypy_options = []
             pypy_options.extend(
                 ['--withmod-%s' % mod for mod in regrtest.usemodules])
-            sopt = " ".join(pypy_options) 
-            cmd = "%s %s %d %s -S %s %s %s -v" %(
-                python, alarm_script, TIMEOUT, 
-                pypy_script, sopt, 
+            sopt = " ".join(pypy_options)
+            cmd = "%s %s %d %s -S %s %s %s -v" % (
+                python, alarm_script, TIMEOUT,
+                pypy_script, sopt,
                 regrrun, fspath.purebasename)
-        return cmd 
+        return cmd
 
-    def runtest(self): 
-        """ invoke a subprocess running the test file via PyPy. 
-            record its output into the 'result/user@host' subdirectory. 
-            (we might want to create subdirectories for 
-            each user, because we will probably all produce 
+    def runtest(self):
+        """ invoke a subprocess running the test file via PyPy.
+            record its output into the 'result/user@host' subdirectory.
+            (we might want to create subdirectories for
+            each user, because we will probably all produce
             such result runs and they will not be the same
-            i am afraid. 
-        """ 
+            i am afraid.
+        """
         regrtest = self.parent.regrtest
         if regrtest.skip:
             if regrtest.skip is True:
             else:
                 msg = regrtest.skip
             py.test.skip(msg)
-        (skipped, exit_status, test_stdout,
-                               test_stderr) = self.getresult(regrtest)
+        (skipped, exit_status, test_stdout, test_stderr) = \
+            self.getresult(regrtest)
         if skipped:
             py.test.skip(test_stderr.splitlines()[-1])
         if exit_status:
         out, err = excinfo.value.args
         return out + err
 
-    def getstatusouterr(self, cmd): 
+    def getstatusouterr(self, cmd):
         tempdir = py.test.ensuretemp(self.fspath.basename)
         stdout = tempdir.join(self.fspath.basename) + '.out'
         stderr = tempdir.join(self.fspath.basename) + '.err'
         if sys.platform == 'win32':
-            status = os.system("%s >%s 2>%s" %(cmd, stdout, stderr))
-            if status>=0:
+            status = os.system("%s >%s 2>%s" % (cmd, stdout, stderr))
+            if status >= 0:
                 status = status
             else:
                 status = 'abnormal termination 0x%x' % status
                 stdout.write('')
                 stderr.write('')
             else:
-                status = os.system("%s >>%s 2>>%s" %(cmd, stdout, stderr))
+                status = os.system("%s >>%s 2>>%s" % (cmd, stdout, stderr))
             if os.WIFEXITED(status):
                 status = os.WEXITSTATUS(status)
             else:
                 status = 'abnormal termination 0x%x' % status
         return status, stdout.read(mode='rU'), stderr.read(mode='rU')
 
-    def getresult(self, regrtest): 
+    def getresult(self, regrtest):
         cmd = self.getinvocation(regrtest)
         tempdir = py.test.ensuretemp(self.fspath.basename)
         oldcwd = tempdir.chdir()
         exit_status, test_stdout, test_stderr = self.getstatusouterr(cmd)
         oldcwd.chdir()
         skipped = False
-        timedout = test_stderr.rfind(26*"=" + "timedout" + 26*"=") != -1 
-        if not timedout: 
+        timedout = test_stderr.rfind(26*"=" + "timedout" + 26*"=") != -1
+        if not timedout:
             timedout = test_stderr.rfind("KeyboardInterrupt") != -1
         if test_stderr.rfind(26*"=" + "skipped" + 26*"=") != -1:
             skipped = True
             # test in test_zipimport_support.py
             if re.search(r'\bFAIL\b', test_stdout) or re.search('[^:]ERROR', test_stderr):
                 outcome = 'FAIL'
-                exit_status = 2  
-        elif timedout: 
-            outcome = "T/O"    
-        else: 
+                exit_status = 2
+        elif timedout:
+            outcome = "T/O"
+        else:
             outcome = "ERR"
-        
+
         return skipped, exit_status, test_stdout, test_stderr
 
     def _keywords(self):
         if regrtest.core:
             lst.append('core')
         return lst
-

lib_pypy/_collections.py

 # Note that PyPy also contains a built-in module '_collections' which will hide
 # this one if compiled in.
 
-import operator
 try:
     from threading import _get_ident as _thread_ident
 except ImportError:

lib_pypy/_ctypes/function.py

 import _ffi
 import sys
 import traceback
-import warnings
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
             return
 
         if argtypes is None:
-            warnings.warn('C function without declared arguments called',
-                          RuntimeWarning, stacklevel=2)
+            # XXX this warning was originally meaning "it's going to be
+            # really slow".  Now we don't worry that much about slowness
+            # of ctypes, and it's strange to get warnings for perfectly-
+            # legal code.
+            #warnings.warn('C function without declared arguments called',
+            #              RuntimeWarning, stacklevel=2)
             argtypes = []
 
         if self._com_index:
         if not outargs:
             return result
 
+        from ctypes import c_void_p
         simple_cdata = type(c_void_p()).__bases__[0]
         outargs = [x.value if type(x).__bases__[0] is simple_cdata else x
                    for x in outargs]

lib_pypy/_curses.py

     if not lib._m_NetBSD:
         for key in range(lib.KEY_MIN, lib.KEY_MAX):
             key_n = lib.keyname(key)
-            if key_n == ffi.NULL or ffi.string(key_n) == "UNKNOWN KEY":
+            if key_n == ffi.NULL:
                 continue
-            key_n = ffi.string(key_n).replace('(', '').replace(')', '')
+            key_n = ffi.string(key_n)
+            if key_n == b"UNKNOWN KEY":
+                continue
+            if not isinstance(key_n, str):   # python 3
+                key_n = key_n.decode()
+            key_n = key_n.replace('(', '').replace(')', '')
             globals()[key_n] = key
 
 _setup()

lib_pypy/_functools.py

         if self.keywords is not None:
             fkeywords = dict(self.keywords, **fkeywords)
         return self.func(*(self.args + fargs), **fkeywords)
+
+    def __reduce__(self):
+        d = dict((k, v) for k, v in self.__dict__.iteritems() if k not in
+                ('func', 'args', 'keywords'))
+        if len(d) == 0:
+            d = None
+        return (type(self), (self.func,),
+                (self.func, self.args, self.keywords, d))
+
+    def __setstate__(self, state):
+        self.func, self.args, self.keywords, d = state
+        if d is not None:
+            self.__dict__.update(d)

lib_pypy/_pypy_interact.py

     import code
     if mainmodule is None:
         import __main__ as mainmodule
-    console = code.InteractiveConsole(mainmodule.__dict__)
+    console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
     # some parts of code.py are copied here because it seems to be impossible
     # to start an interactive console without printing at least one line
     # of banner
 # ____________________________________________________________
 
 if __name__ == '__main__':    # for testing
-    import os
     if os.getenv('PYTHONSTARTUP'):
         execfile(os.getenv('PYTHONSTARTUP'))
     interactive_console()

lib_pypy/_sha512.py

 
 def sha_transform(sha_info):
     W = []
-    
+
     d = sha_info['data']
     for i in xrange(0,16):
         W.append( (d[8*i]<<56) + (d[8*i+1]<<48) + (d[8*i+2]<<40) + (d[8*i+3]<<32) + (d[8*i+4]<<24) + (d[8*i+5]<<16) + (d[8*i+6]<<8) + d[8*i+7])
-    
+
     for i in xrange(16,80):
         W.append( (Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xffffffffffffffff )
-    
+
     ss = sha_info['digest'][:]
-    
+
     def RND(a,b,c,d,e,f,g,h,i,ki):
         t0 = (h + Sigma1(e) + Ch(e, f, g) + ki + W[i]) & 0xffffffffffffffff
         t1 = (Sigma0(a) + Maj(a, b, c)) & 0xffffffffffffffff
         d = (d + t0) & 0xffffffffffffffff
         h = (t0 + t1) & 0xffffffffffffffff
         return d & 0xffffffffffffffff, h & 0xffffffffffffffff
-    
+
     ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],0,0x428a2f98d728ae22)
     ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],1,0x7137449123ef65cd)
     ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],2,0xb5c0fbcfec4d3b2f)
     ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],77,0x597f299cfc657e2a)
     ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],78,0x5fcb6fab3ad6faec)
     ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],79,0x6c44198c4a475817)
-    
-    
+
     dig = []
     for i, x in enumerate(sha_info['digest']):
         dig.append( (x + ss[i]) & 0xffffffffffffffff )
     if clo < sha_info['count_lo']:
         sha_info['count_hi'] += 1
     sha_info['count_lo'] = clo
-    
+
     sha_info['count_hi'] += (count >> 29)
-    
+
     if sha_info['local']:
         i = SHA_BLOCKSIZE - sha_info['local']
         if i > count:
             i = count
-        
+
         # copy buffer
         for x in enumerate(buffer[buffer_idx:buffer_idx+i]):
             sha_info['data'][sha_info['local']+x[0]] = struct.unpack('B', x[1])[0]
-        
+
         count -= i
         buffer_idx += i
-        
+
         sha_info['local'] += i
         if sha_info['local'] == SHA_BLOCKSIZE:
             sha_transform(sha_info)
             sha_info['local'] = 0
         else:
             return
-    
+
     while count >= SHA_BLOCKSIZE:
         # copy buffer
         sha_info['data'] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + SHA_BLOCKSIZE]]
         count -= SHA_BLOCKSIZE
         buffer_idx += SHA_BLOCKSIZE
         sha_transform(sha_info)
-    
-    
+
     # copy buffer
     pos = sha_info['local']
     sha_info['data'][pos:pos+count] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + count]]
         sha_info['data'] = [0] * SHA_BLOCKSIZE
     else:
         sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count))
-    
+
     sha_info['data'][112] = 0;
     sha_info['data'][113] = 0;
     sha_info['data'][114] = 0;
     sha_info['data'][117] = 0;
     sha_info['data'][118] = 0;
     sha_info['data'][119] = 0;
-    
+
     sha_info['data'][120] = (hi_bit_count >> 24) & 0xff
     sha_info['data'][121] = (hi_bit_count >> 16) & 0xff
     sha_info['data'][122] = (hi_bit_count >>  8) & 0xff
     sha_info['data'][125] = (lo_bit_count >> 16) & 0xff
     sha_info['data'][126] = (lo_bit_count >>  8) & 0xff
     sha_info['data'][127] = (lo_bit_count >>  0) & 0xff
-    
+
     sha_transform(sha_info)
-    
+
     dig = []
     for i in sha_info['digest']:
         dig.extend([ ((i>>56) & 0xff), ((i>>48) & 0xff), ((i>>40) & 0xff), ((i>>32) & 0xff), ((i>>24) & 0xff), ((i>>16) & 0xff), ((i>>8) & 0xff), (i & 0xff) ])
         self._sha = sha_init()
         if s:
             sha_update(self._sha, getbuf(s))
-    
+
     def update(self, s):
         sha_update(self._sha, getbuf(s))
-    
+
     def digest(self):
         return sha_final(self._sha.copy())[:self._sha['digestsize']]
-    
+
     def hexdigest(self):
         return ''.join(['%.2x' % ord(i) for i in self.digest()])
 
         return new
 
 def test():
+    import _sha512
+
     a_str = "just a test string"
-    
+
     assert _sha512.sha512().hexdigest() == sha512().hexdigest()
     assert _sha512.sha512(a_str).hexdigest() == sha512(a_str).hexdigest()
     assert _sha512.sha512(a_str*7).hexdigest() == sha512(a_str*7).hexdigest()
-    
+
     s = sha512(a_str)
     s.update(a_str)
     assert _sha512.sha512(a_str+a_str).hexdigest() == s.hexdigest()

lib_pypy/_sqlite3.py

 import string
 import sys
 import weakref
-import array
 from threading import _get_ident as _thread_get_ident
 try:
     from __pypy__ import newlist_hint
     const char **pzTail     /* OUT: Pointer to unused portion of zSql */
 );
 
-void sqlite3_result_blob(sqlite3_context*, const char*, int, void(*)(void*));
+void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*));
 void sqlite3_result_double(sqlite3_context*, double);
 void sqlite3_result_error(sqlite3_context*, const char*, int);
 void sqlite3_result_error16(sqlite3_context*, const void*, int);
     typedef ... sqlite3;
     int sqlite3_enable_load_extension(sqlite3 *db, int onoff);
     """)
-    unverified_lib = unverified_ffi.dlopen('sqlite3')
+    libname = 'sqlite3'
+    if sys.platform == 'win32':
+        import os
+        _libname = os.path.join(os.path.dirname(sys.executable), libname)
+        if os.path.exists(_libname + '.dll'):
+            libname = _libname
+    unverified_lib = unverified_ffi.dlopen(libname)
     return hasattr(unverified_lib, 'sqlite3_enable_load_extension')
 
 if _has_load_extension():
         if isinstance(name, unicode):
             name = name.encode('utf-8')
         ret = _lib.sqlite3_create_collation(self._db, name,
-                                              _lib.SQLITE_UTF8,
-                                              _ffi.NULL,
-                                              collation_callback)
+                                            _lib.SQLITE_UTF8,
+                                            _ffi.NULL,
+                                            collation_callback)
         if ret != _lib.SQLITE_OK:
             raise self._get_exception(ret)
 
                     return _lib.SQLITE_DENY
             self.__func_cache[callback] = authorizer
 
-        ret = _lib.sqlite3_set_authorizer(self._db,
-                                          authorizer,
-                                          _ffi.NULL)
+        ret = _lib.sqlite3_set_authorizer(self._db, authorizer, _ffi.NULL)
         if ret != _lib.SQLITE_OK:
             raise self._get_exception(ret)
 
                 @_ffi.callback("int(void*)")
                 def progress_handler(userdata):
                     try:
-                        ret = callable()
-                        return bool(ret)
+                        return bool(callable())
                     except Exception:
                         # abort query if error occurred
                         return 1
                 self.__func_cache[callable] = progress_handler
-        _lib.sqlite3_progress_handler(self._db, nsteps,
-                                            progress_handler,
-                                            _ffi.NULL)
+        _lib.sqlite3_progress_handler(self._db, nsteps, progress_handler,
+                                      _ffi.NULL)
 
     if sys.version_info[0] >= 3:
         def __get_in_transaction(self):
         self._reset = False
         self.__locked = False
         self.__closed = False
-        self.__description = None
+        self.__lastrowid = None
         self.__rowcount = -1
 
         con._check_thread()
     def __check_reset(self):
         if self._reset:
             raise InterfaceError(
-                    "Cursor needed to be reset because of commit/rollback "
-                    "and can no longer be fetched from.")
+                "Cursor needed to be reset because of commit/rollback "
+                "and can no longer be fetched from.")
 
     def __build_row_cast_map(self):
         if not self.__connection._detect_types:
                 elif typ == _lib.SQLITE_BLOB:
                     blob = _lib.sqlite3_column_blob(self.__statement._statement, i)
                     blob_len = _lib.sqlite3_column_bytes(self.__statement._statement, i)
-                    # make a copy of the data into an array, in order to get
-                    # a read-write buffer in the end, and one that own the
-                    # memory for a more predictable length of time than 'blob'
-                    copy = array.array("c", _ffi.buffer(blob, blob_len))
-                    val = _BLOB_TYPE(copy)
+                    val = _BLOB_TYPE(_ffi.buffer(blob, blob_len)[:])
             row.append(val)
         return tuple(row)
 
         try:
             if not isinstance(sql, basestring):
                 raise ValueError("operation parameter must be str or unicode")
-            self.__description = None
+            try:
+                del self.__description
+            except AttributeError:
+                pass
             self.__rowcount = -1
             self.__statement = self.__connection._statement_cache.get(sql)
 
             if self.__connection._isolation_level is not None:
-                if self.__statement._kind == Statement._DDL:
+                if self.__statement._type in ("UPDATE", "DELETE", "INSERT", "REPLACE"):
+                    if not self.__connection._in_transaction:
+                        self.__connection._begin()
+                elif self.__statement._type == "OTHER":
                     if self.__connection._in_transaction:
                         self.__connection.commit()
-                elif self.__statement._kind == Statement._DML:
-                    if not self.__connection._in_transaction:
-                        self.__connection._begin()
-
-            if multiple and self.__statement._kind != Statement._DML:
-                raise ProgrammingError("executemany is only for DML statements")
+                elif self.__statement._type == "SELECT":
+                    if multiple:
+                        raise ProgrammingError("You cannot execute SELECT "
+                                               "statements in executemany().")
 
             for params in many_params:
                 self.__statement._set_params(params)
 
                 # Actually execute the SQL statement
                 ret = _lib.sqlite3_step(self.__statement._statement)
-                if ret not in (_lib.SQLITE_DONE, _lib.SQLITE_ROW):
+
+                if ret == _lib.SQLITE_ROW:
+                    if multiple:
+                        raise ProgrammingError("executemany() can only execute DML statements.")
+                    self.__build_row_cast_map()
+                    self.__next_row = self.__fetch_one_row()
+                elif ret == _lib.SQLITE_DONE:
+                    if not multiple:
+                        self.__statement._reset()
+                else:
                     self.__statement._reset()
                     raise self.__connection._get_exception(ret)
 
-                if self.__statement._kind == Statement._DML:
-                    self.__statement._reset()
-
-                if ret == _lib.SQLITE_ROW:
-                    self.__build_row_cast_map()
-                    self.__next_row = self.__fetch_one_row()
-
-                if self.__statement._kind == Statement._DML:
+                if self.__statement._type in ("UPDATE", "DELETE", "INSERT", "REPLACE"):
                     if self.__rowcount == -1:
                         self.__rowcount = 0
                     self.__rowcount += _lib.sqlite3_changes(self.__connection._db)
+
+                if not multiple and self.__statement._type == "INSERT":
+                    self.__lastrowid = _lib.sqlite3_last_insert_rowid(self.__connection._db)
+                else:
+                    self.__lastrowid = None
+
+                if multiple:
+                    self.__statement._reset()
         finally:
             self.__connection._in_transaction = \
                 not _lib.sqlite3_get_autocommit(self.__connection._db)
         try:
             next_row = self.__next_row
         except AttributeError:
-            self.__statement._reset()
             raise StopIteration
         del self.__next_row
 
             next_row = self.row_factory(self, next_row)
 
         ret = _lib.sqlite3_step(self.__statement._statement)
-        if ret not in (_lib.SQLITE_DONE, _lib.SQLITE_ROW):
+        if ret == _lib.SQLITE_ROW:
+            self.__next_row = self.__fetch_one_row()
+        else:
             self.__statement._reset()
-            raise self.__connection._get_exception(ret)
-        elif ret == _lib.SQLITE_ROW:
-            self.__next_row = self.__fetch_one_row()
+            if ret != _lib.SQLITE_DONE:
+                raise self.__connection._get_exception(ret)
         return next_row
 
     if sys.version_info[0] < 3:
     rowcount = property(__get_rowcount)
 
     def __get_description(self):
-        if self.__description is None:
+        try:
+            return self.__description
+        except AttributeError:
             self.__description = self.__statement._get_description()
-        return self.__description
+            return self.__description
     description = property(__get_description)
 
     def __get_lastrowid(self):
-        return _lib.sqlite3_last_insert_rowid(self.__connection._db)
+        return self.__lastrowid
     lastrowid = property(__get_lastrowid)
 
     def setinputsizes(self, *args):
 
 
 class Statement(object):
-    _DML, _DQL, _DDL = range(3)
-
     _statement = None
 
     def __init__(self, connection, sql):
 
         if not isinstance(sql, basestring):
             raise Warning("SQL is of wrong type. Must be string or unicode.")
-        first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper()
-        if first_word in ("INSERT", "UPDATE", "DELETE", "REPLACE"):
-            self._kind = Statement._DML
-        elif first_word in ("SELECT", "PRAGMA"):
-            self._kind = Statement._DQL
+
+        first_word = sql.lstrip().split(" ")[0].upper()
+        if first_word == "":
+            self._type = "INVALID"
+        elif first_word in ("SELECT", "INSERT", "UPDATE", "DELETE", "REPLACE"):
+            self._type = first_word
         else:
-            self._kind = Statement._DDL
+            self._type = "OTHER"
 
         if isinstance(sql, unicode):
             sql = sql.encode('utf-8')
 
         if ret == _lib.SQLITE_OK and not self._statement:
             # an empty statement, work around that, as it's the least trouble
+            self._type = "SELECT"
             c_sql = _ffi.new("char[]", b"select 42")
             ret = _lib.sqlite3_prepare_v2(self.__con._db, c_sql, -1,
                                           statement_star, next_char)
             self._statement = statement_star[0]
-            self._kind = Statement._DQL
+
         if ret != _lib.SQLITE_OK:
             raise self.__con._get_exception(ret)
 
             raise ValueError("parameters are of unsupported type")
 
     def _get_description(self):
-        if self._kind == Statement._DML:
+        if self._type in ("INSERT", "UPDATE", "DELETE", "REPLACE"):
             return None
         desc = []
         for i in xrange(_lib.sqlite3_column_count(self._statement)):
         elif typ == _lib.SQLITE_BLOB:
             blob = _lib.sqlite3_value_blob(params[i])
             blob_len = _lib.sqlite3_value_bytes(params[i])
-            val = _BLOB_TYPE(_ffi.buffer(blob, blob_len))
+            val = _BLOB_TYPE(_ffi.buffer(blob, blob_len)[:])
         else:
             raise NotImplementedError
         _params.append(val)
             'c': os.O_RDWR | os.O_CREAT,
             'n': os.O_RDWR | os.O_CREAT | os.O_TRUNC,
             }[flag]
-    except KeyError, e:
+    except KeyError:
         raise error("arg 2 to open should be 'r', 'w', 'c', or 'n'")
 
     a_db = getattr(lib, funcs['open'])(filename, openflag, mode)

lib_pypy/greenlet.py

-import _continuation, sys
+import _continuation
 
 __version__ = "0.4.0"
 

lib_pypy/pyrepl/python_reader.py

 
     def execute(self, text):
         try:
-            # ooh, look at the hack:            
+            # ooh, look at the hack:
             code = self.compile("# coding:utf8\n"+text.encode('utf-8'),
-                                '<input>', 'single')
+                                '<stdin>', 'single')
         except (OverflowError, SyntaxError, ValueError):
-            self.showsyntaxerror("<input>")
+            self.showsyntaxerror('<stdin>')
         else:
             self.runcode(code)
             if sys.stdout and not sys.stdout.closed:

lib_pypy/pyrepl/simple_interact.py

     import code
     if mainmodule is None:
         import __main__ as mainmodule
-    console = code.InteractiveConsole(mainmodule.__dict__)
+    console = code.InteractiveConsole(mainmodule.__dict__, filename='<stdin>')
 
     def more_lines(unicodetext):
         # ooh, look at the hack:
         src = "#coding:utf-8\n"+unicodetext.encode('utf-8')
         try:
-            code = console.compile(src, '<input>', 'single')
+            code = console.compile(src, '<stdin>', 'single')
         except (OverflowError, SyntaxError, ValueError):
             return False
         else:
 
 ::
 
-  pypy <path to rpython>/rpython/bin/rpython -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=armv7 targetpypystandalone.py
+  pypy <path to rpython>/rpython/bin/rpython -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py
 
 The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_
 

pypy/doc/config/translation.gcrootfinder.txt

 - ``--gcrootfinder=asmgcc``: use assembler hackery to find the
   roots directly from the normal stack.  This is a bit faster,
   but platform specific.  It works so far with GCC or MSVC,
-  on i386 and x86-64.
+  on i386 and x86-64.  It is tested only on Linux (where it is
+  the default) so other platforms (as well as MSVC) may need
+  various fixes before they can be used.
 
 You may have to force the use of the shadowstack root finder if
 you are running into troubles or if you insist on translating

pypy/doc/dir-reference.rst

+PyPy directory cross-reference 
+------------------------------
+
+Here is a fully referenced alphabetical two-level deep 
+directory overview of PyPy: 
+
+=================================  ============================================
+Directory                          explanation/links
+=================================  ============================================
+`pypy/bin/`_                       command-line scripts, mainly
+                                   `pypy/bin/pyinteractive.py`_
+
+`pypy/config/`_                    handles the numerous options for building
+                                   and running PyPy
+
+`pypy/doc/`_                       text versions of PyPy developer
+                                   documentation
+
+`pypy/doc/config/`_                documentation for the numerous translation
+                                   options
+
+`pypy/doc/discussion/`_            drafts of ideas and documentation
+
+``doc/*/``                         other specific documentation topics or tools
+
+`pypy/interpreter/`_               `bytecode interpreter`_ and related objects
+                                   (frames, functions, modules,...) 
+
+`pypy/interpreter/pyparser/`_      interpreter-level Python source parser
+
+`pypy/interpreter/astcompiler/`_   interpreter-level bytecode compiler,
+                                   via an AST representation
+
+`pypy/module/`_                    contains `mixed modules`_
+                                   implementing core modules with 
+                                   both application and interpreter level code.
+                                   Not all are finished and working.  Use
+                                   the ``--withmod-xxx``
+                                   or ``--allworkingmodules`` translation
+                                   options.
+
+`pypy/objspace/`_                  `object space`_ implementations
+
+`pypy/objspace/std/`_              the StdObjSpace_ implementing CPython's
+                                   objects and types
+
+`pypy/tool/`_                      various utilities and hacks used
+                                   from various places 
+
+`pypy/tool/algo/`_                 general-purpose algorithmic and mathematic
+                                   tools
+
+`pypy/tool/pytest/`_               support code for our `testing methods`_
+
+
+`rpython/annotator/`_              `type inferencing code`_ for
+                                   `RPython`_ programs 
+
+`rpython/config/`_                 handles the numerous options for RPython
+
+
+`rpython/flowspace/`_              the FlowObjSpace_ implementing
+                                   `abstract interpretation`_
+
+`rpython/rlib/`_                   a `"standard library"`_ for RPython_
+                                   programs
+
+`rpython/rtyper/`_                 the `RPython Typer`_ 
+
+`rpython/rtyper/lltypesystem/`_    the `low-level type system`_ for
+                                   C-like backends
+
+`rpython/rtyper/ootypesystem/`_    the `object-oriented type system`_
+                                   for OO backends
+
+`rpython/memory/`_                 the `garbage collector`_ construction
+                                   framework
+
+`rpython/translator/`_             translation_ backends and support code
+
+`rpython/translator/backendopt/`_  general optimizations that run before a 
+                                   backend generates code
+
+`rpython/translator/c/`_           the `GenC backend`_, producing C code
+                                   from an
+                                   RPython program (generally via the rtyper_)
+
+`rpython/translator/cli/`_         the `CLI backend`_ for `.NET`_
+                                   (Microsoft CLR or Mono_)
+
+`pypy/goal/`_                      our `main PyPy-translation scripts`_
+                                   live here
+
+`rpython/translator/jvm/`_         the Java backend
+
+`rpython/translator/tool/`_        helper tools for translation
+
+`dotviewer/`_                      `graph viewer`_
+
+``*/test/``                        many directories have a test subdirectory
+                                   containing test 
+                                   modules (see `Testing in PyPy`_) 
+
+``_cache/``                        holds cache files from various purposes
+=================================  ============================================
+
+.. _`bytecode interpreter`: interpreter.html
+.. _`Testing in PyPy`: coding-guide.html#testing-in-pypy 
+.. _`mixed modules`: coding-guide.html#mixed-modules 
+.. _`modules`: coding-guide.html#modules 
+.. _`basil`: http://people.cs.uchicago.edu/~jriehl/BasilTalk.pdf
+.. _`object space`: objspace.html
+.. _FlowObjSpace: objspace.html#the-flow-object-space 
+.. _`transparent proxies`: objspace-proxies.html#tproxy
+.. _`Differences between PyPy and CPython`: cpython_differences.html
+.. _`What PyPy can do for your objects`: objspace-proxies.html
+.. _`Continulets and greenlets`: stackless.html
+.. _StdObjSpace: objspace.html#the-standard-object-space 
+.. _`abstract interpretation`: http://en.wikipedia.org/wiki/Abstract_interpretation
+.. _`rpython`: coding-guide.html#rpython 
+.. _`type inferencing code`: translation.html#the-annotation-pass 
+.. _`RPython Typer`: translation.html#rpython-typer 
+.. _`testing methods`: coding-guide.html#testing-in-pypy
+.. _`translation`: translation.html 
+.. _`GenC backend`: translation.html#genc 
+.. _`CLI backend`: cli-backend.html
+.. _`py.py`: getting-started-python.html#the-py.py-interpreter
+.. _`translatorshell.py`: getting-started-dev.html#try-out-the-translator
+.. _JIT: jit/index.html
+.. _`JIT Generation in PyPy`: jit/index.html
+.. _`just-in-time compiler generator`: jit/index.html
+.. _rtyper: rtyper.html
+.. _`low-level type system`: rtyper.html#low-level-type
+.. _`object-oriented type system`: rtyper.html#oo-type
+.. _`garbage collector`: garbage_collection.html
+.. _`main PyPy-translation scripts`: getting-started-python.html#translating-the-pypy-python-interpreter
+.. _`.NET`: http://www.microsoft.com/net/
+.. _Mono: http://www.mono-project.com/
+.. _`"standard library"`: rlib.html
+.. _`graph viewer`: getting-started-dev.html#try-out-the-translator

pypy/doc/getting-started-dev.rst

-===============================================================================
-Getting Started with the Translation Toolchain and Development Process
-===============================================================================
+============================
+Getting Started with RPython
+============================
 
 .. contents::
 
+RPython is a subset of Python that can be statically compiled. The PyPy
+interpreter is written mostly in RPython (with pieces in Python), while
+the RPython compiler is written in Python. The hard to understand part
+is that Python is a meta-programming language for RPython, that is,
+RPython is considered from live objects **after** the imports are done.
+This might require more explanation. You start writing RPython from
+``entry_point``, a good starting point is
+``rpython/translator/goal/targetnopstandalone.py``. This does not do all that
+much, but is a start. Now if code analyzed (in this case ``entry_point``)
+calls some functions, those calls will be followed. Those followed calls
+have to be RPython themselves (and everything they call etc.), however not
+entire module files. To show how you can use metaprogramming, we can do
+a silly example (note that closures are not RPython)::
+
+  def generator(operation):
+      if operation == 'add':
+         def f(a, b):
+             return a + b
+      else:
+         def f(a, b):
+             return a - b
+      return f
+
+  add = generator('add')
+  sub = generator('sub')
+
+  def entry_point(argv):
+      print add(sub(int(argv[1]), 3) 4)
+      return 0
+
+In this example ``entry_point`` is RPython,  ``add`` and ``sub`` are RPython,
+however, ``generator`` is not.
+
+A good introductory level articles are available:
+
+* Laurence Tratt -- `Fast Enough VMs in Fast Enough Time`_.
+
+* `How to write interpreters in RPython`_ and `part 2`_ by Andrew Brown.
+
+.. _`Fast Enough VMs in Fast Enough Time`: http://tratt.net/laurie/tech_articles/articles/fast_enough_vms_in_fast_enough_time
+
+.. _`How to write interpreters in RPython`: http://morepypy.blogspot.com/2011/04/tutorial-writing-interpreter-with-pypy.html
+
+.. _`part 2`: http://morepypy.blogspot.com/2011/04/tutorial-part-2-adding-jit.html
+
 .. _`try out the translator`:
 
 Trying out the translator
-------------------------- 
+-------------------------
 
 The translator is a tool based on the PyPy interpreter which can translate
 sufficiently static RPython programs into low-level code (in particular it can
 
     >>> t = Translation(snippet.is_perfect_number, [int])
     >>> t.view()
-        
+
 After that, the graph viewer pops up, that lets you interactively inspect the
 flow graph. To move around, click on something that you want to inspect.
 To get help about how to use it, press 'H'. To close it again, press 'Q'.
 The object returned by ``compile_cli`` or ``compile_jvm``
 is a wrapper around the real
 executable: the parameters are passed as command line arguments, and
-the returned value is read from the standard output.  
+the returned value is read from the standard output.
 
 Once you have compiled the snippet, you can also try to launch the
-executable directly from the shell. You will find the 
+executable directly from the shell. You will find the
 executable in one of the ``/tmp/usession-*`` directories::
 
     # For CLI:

pypy/doc/getting-started-python.rst

 
 
 PyPy's Python interpreter is a very compliant Python
-interpreter implemented in Python.  When translated to C, it passes most of 
+interpreter implemented in RPython.  When compiled, it passes most of 
 `CPythons core language regression tests`_ and comes with many of the extension
 modules included in the standard library including ``ctypes``. It can run large
 libraries such as Django_ and Twisted_. There are some small behavioral
 Translating using the CLI backend
 +++++++++++++++++++++++++++++++++
 
+**Note: the CLI backend is no longer maintained**
+
 To create a standalone .NET executable using the `CLI backend`_::
 
     ./translate.py --backend=cli targetpypystandalone.py

pypy/doc/how-to-contribute.rst

+How to contribute to PyPy
+-------------------------
+
+This page describes how to contribute to the PyPy project. The first thing
+to remember is that PyPy project is very different than most projects out there.
+It's also different from a classic compiler project, so academic courses
+about compilers often don't apply or lead in the wrong direction.
+
+Don't just hack
+---------------
+
+The first and most important rule how not to contribute to PyPy is
+"just hacking". This won't work. There are two major reasons why not
+-- build times are large and PyPy has very thick layer separation which
+make it harder to "just hack a feature".
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+Layers
+------
+
+PyPy has layers. Those layers help us keep the respective parts separated enough
+to be worked on independently and make the complexity manageable. This is,
+again, just a sanity requirement for such a complex project. For example writing
+a new optimization for the JIT usually does **not** involve touching a Python
+interpreter at all or the JIT assembler backend or the garbage collector.
+Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+The short list of layers for further reading. For each of those layers, a good
+entry point is a test subdirectory in respective directories. It usually
+describes (better or worse) the interfaces between the submodules. For the
+``pypy`` subdirectory, most tests are small snippets of python programs that
+check for correctness (calls ``AppTestXxx``) that will call the appropriate
+part of the interpreter. For the ``rpython`` directory, most tests are small
+RPython interpreters that perform certain tasks. To see how they translate
+to low-level graphs, run them with ``--view``. To see small interpreters
+with a JIT compiler, use ``--viewloops`` option.
+
+* **python interpreter** - it's the part implemented in the ``pypy/`` directory.
+  It's implemented in RPython, which is a high level static language with
+  classes, garbage collection, just-in-time compiler generation and the ability
+  to call C. A cool part about it is that it can be run untranslated, so all
+  the tests are runnable without translating PyPy.
+
+  **interpreter** contains the interpreter core
+
+  **objspace** contains implementations of various objects exported to
+  the Python layer
+
+  **module** directory contains extension modules written in RPython
+
+* **rpython compiler** that resides in ``rpython/annotator`` and
+  ``rpython/rtyper`` directories. Consult `introduction to RPython`_ for
+  further reading
+
+* **JIT generator** lives in ``rpython/jit`` directory. optimizations live
+  in ``rpython/jit/metainterp/optimizeopt``, the main JIT in
+  ``rpython/jit/metainterp`` (runtime part) and
+  ``rpython/jit/codewriter`` (translation-time part). Backends live in
+  ``rpython/jit/backend``.
+
+* **garbage collection** lives in ``rpython/memory``
+
+The rest of directories serve specific niche goal and are unlikely a good
+entry point.
+
+.. _`introduction to RPython`: getting-started-dev.rst

pypy/doc/index.rst

 
-Welcome to PyPy Development
-=============================================
+Welcome to PyPy
+===============
 
-The PyPy project aims at producing a flexible and fast Python_
-implementation.  The guiding idea is to translate a Python-level
-description of the Python language itself to lower level languages.
-Rumors have it that the secret goal is being faster-than-C which is
-nonsense, isn't it?  `more...`_
+The PyPy project aims to produce a flexible and fast Python_
+implementation.  This page documents the development of the PyPy
+project itself. If you don't know what PyPy is, consult the `PyPy
+website`_. If you just want to use PyPy, consult the `download`_ page
+and the `getting started with pypy`_ documents. If you want to help
+develop PyPy -- keep reading!
 
-Getting into PyPy ... 
-=============================================
+PyPy is written in a language called `RPython`_, which is suitable for
+writing dynamic language interpreters (and not much else). RPython is
+a subset of Python and is itself written in Python.  If you'd like to
+learn more about RPython, `Starting with RPython`_ should provide a
+reasonable overview.
+
+**If you would like to contribute to PyPy**, please read `how to
+contribute`_ first.  PyPy's development style is somewhat different to
+that of many other software projects and it often surprises
+newcomers. What is **not** necessary is an academic background from
+university in writing compilers -- much of it does not apply to PyPy
+any way.
+
+All of the documentation and source code is available under the MIT license,
+unless otherwise specified. Consult `LICENSE`_
+
+.. _`download`: http://pypy.org/download.html
+.. _`getting started with pypy`: getting-started-python.html
+.. _`RPython`: coding-guide.html#RPython
+.. _`Starting with RPython`: getting-started-dev.html
+.. _`how to contribute`: how-to-contribute.html
+.. _`PyPy website`: http://pypy.org
+.. _`LICENSE`: https://bitbucket.org/pypy/pypy/src/default/LICENSE
+
+Index of various topics:
+========================
 
 * `Getting started`_: how to install and run the PyPy Python interpreter
 
 
 * `potential project ideas`_: In case you want to get your feet wet...
 
+* `more stuff`_: this is a collection of documentation that's there, but not
+  particularly organized 
+
 Documentation for the PyPy Python Interpreter
-===============================================
+=============================================
 
 New features of PyPy's Python Interpreter and 
 Translation Framework: 
 
   * `Differences between PyPy and CPython`_
-  * `What PyPy can do for your objects`_
-  * `Continulets and greenlets`_
+  * `What PyPy can do for your objects`_ - transparent proxy documentation
+  * `Continulets and greenlets`_ - documentation about stackless features
   * `JIT Generation in PyPy`_ 
+  * `JIT hooks`_
   * `Sandboxing Python code`_
+  * `Garbage collection environment variables`_
 
-Status_ of the project.
-
+.. _`Differences between PyPy and CPython`: cpython_differences.html
+.. _`What PyPy can do for your objects`: objspace-proxies.html
+.. _`Continulets and greenlets`: stackless.html
+.. _`JIT Generation in PyPy`: jit/index.html
+.. _`JIT hooks`: jit-hooks.html
+.. _`Sandboxing Python code`: sandbox.html
+.. _`Garbage collection environment variables`: gc_info.html
 
 Mailing lists, bug tracker, IRC channel
 =============================================
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html
-
-Project Documentation
-=====================================
-
-PyPy was funded by the EU for several years. See the `web site of the EU
-project`_ for more details.
-
-.. _`web site of the EU project`: http://pypy.org
-
-architecture_ gives a complete view of PyPy's basic design. 
-
-`coding guide`_ helps you to write code for PyPy (especially also describes
-coding in RPython a bit). 
-
-`sprint reports`_ lists reports written at most of our sprints, from
-2003 to the present.
-
-`papers, talks and related projects`_ lists presentations 
-and related projects as well as our published papers.
-
-`PyPy video documentation`_ is a page linking to the videos (e.g. of talks and
-introductions) that are available.
-
-`Technical reports`_ is a page that contains links to the
-reports that we submitted to the European Union.
-
-`development methodology`_ describes our sprint-driven approach.
-
-`LICENSE`_ contains licensing details (basically a straight MIT-license). 
-
-`Glossary`_ of PyPy words to help you align your inner self with
-the PyPy universe.
-
-
-Status
-===================================
-
-PyPy can be used to run Python programs on Linux, OS/X,
-Windows, on top of .NET, and on top of Java.
-To dig into PyPy it is recommended to try out the current
-Mercurial default branch, which is always working or mostly working,
-instead of the latest release, which is `2.0 beta1`__.
-
-.. __: release-2.0.0-beta1.html
-
-PyPy is mainly developed on Linux and Mac OS X.  Windows is supported,
-but platform-specific bugs tend to take longer before we notice and fix
-them.  Linux 64-bit machines are supported (though it may also take some
-time before we notice and fix bugs).
-
-PyPy's own tests `summary`_, daily updated, run through BuildBot infrastructure.
-You can also find CPython's compliance tests run with compiled ``pypy-c``
-executables there.
-
-
-Source Code Documentation
-===============================================
-
-`object spaces`_ discusses the object space interface 
-and several implementations. 
-
-`bytecode interpreter`_ explains the basic mechanisms 
-of the bytecode interpreter and virtual machine. 
-
-`interpreter optimizations`_ describes our various strategies for
-improving the performance of our interpreter, including alternative
-object implementations (for strings, dictionaries and lists) in the
-standard object space.
-
-`translation`_ is a detailed overview of our translation process.  The
-rtyper_ is the largest component of our translation process.
-
-`dynamic-language translation`_ is a paper that describes
-the translation process, especially the flow object space
-and the annotator in detail. (This document is one
-of the `EU reports`_.)
-
-`low-level encapsulation`_ describes how our approach hides
-away a lot of low level details. This document is also part
-of the `EU reports`_.
-
-`translation aspects`_ describes how we weave different
-properties into our interpreter during the translation
-process. This document is also part of the `EU reports`_.
-
-`garbage collector`_ strategies that can be used by the virtual
-machines produced by the translation process.
-
-`parser`_ contains (outdated, unfinished) documentation about
-the parser.
-
-`rlib`_ describes some modules that can be used when implementing programs in
-RPython.
-
-`configuration documentation`_ describes the various configuration options that
-allow you to customize PyPy.
-
-`CLI backend`_ describes the details of the .NET backend.
-
-`JIT Generation in PyPy`_ describes how we produce the Python Just-in-time Compiler
-from our Python interpreter.
-
-
-
-.. _`FAQ`: faq.html
-.. _Glossary: glossary.html
-.. _`PyPy video documentation`: video-index.html
-.. _parser: parser.html
-.. _`development methodology`: dev_method.html
-.. _`sprint reports`: sprint-reports.html
-.. _`papers, talks and related projects`: extradoc.html
-.. _`object spaces`: objspace.html 
-.. _`interpreter optimizations`: interpreter-optimizations.html 
-.. _`translation`: translation.html 
-.. _`dynamic-language translation`: https://bitbucket.org/pypy/extradoc/raw/tip/eu-report/D05.1_Publish_on_translating_a_very-high-level_description.pdf
-.. _`low-level encapsulation`: low-level-encapsulation.html
-.. _`translation aspects`: translation-aspects.html
-.. _`configuration documentation`: config/
-.. _`coding guide`: coding-guide.html 
-.. _`architecture`: architecture.html 
-.. _`getting started`: getting-started.html 
-.. _`bytecode interpreter`: interpreter.html 
-.. _`EU reports`: index-report.html
-.. _`Technical reports`: index-report.html
-.. _`summary`: http://buildbot.pypy.org/summary
-.. _`ideas for PyPy related projects`: project-ideas.html
-.. _`Nightly builds and benchmarks`: http://tuatara.cs.uni-duesseldorf.de/benchmark.html
-.. _`directory reference`: 
-.. _`rlib`: rlib.html
-.. _`Sandboxing Python code`: sandbox.html
-.. _`LICENSE`: https://bitbucket.org/pypy/pypy/src/default/LICENSE
-
-PyPy directory cross-reference 
-------------------------------
-
-Here is a fully referenced alphabetical two-level deep 
-directory overview of PyPy: 
-
-=================================  ============================================
-Directory                          explanation/links
-=================================  ============================================
-`pypy/bin/`_                       command-line scripts, mainly
-                                   `pypy/bin/pyinteractive.py`_
-
-`pypy/config/`_                    handles the numerous options for building
-                                   and running PyPy
-
-`pypy/doc/`_                       text versions of PyPy developer
-                                   documentation
-
-`pypy/doc/config/`_                documentation for the numerous translation
-                                   options
-
-`pypy/doc/discussion/`_            drafts of ideas and documentation
-
-``doc/*/``                         other specific documentation topics or tools
-
-`pypy/interpreter/`_               `bytecode interpreter`_ and related objects
-                                   (frames, functions, modules,...) 
-
-`pypy/interpreter/pyparser/`_      interpreter-level Python source parser
-
-`pypy/interpreter/astcompiler/`_   interpreter-level bytecode compiler,
-                                   via an AST representation
-
-`pypy/module/`_                    contains `mixed modules`_
-                                   implementing core modules with 
-                                   both application and interpreter level code.
-                                   Not all are finished and working.  Use
-                                   the ``--withmod-xxx``
-                                   or ``--allworkingmodules`` translation
-                                   options.
-
-`pypy/objspace/`_                  `object space`_ implementations
-
-`pypy/objspace/std/`_              the StdObjSpace_ implementing CPython's
-                                   objects and types
-
-`pypy/tool/`_                      various utilities and hacks used
-                                   from various places 
-
-`pypy/tool/algo/`_                 general-purpose algorithmic and mathematic
-                                   tools
-
-`pypy/tool/pytest/`_               support code for our `testing methods`_
-
-
-`rpython/annotator/`_              `type inferencing code`_ for
-                                   `RPython`_ programs 
-
-`rpython/config/`_                 handles the numerous options for RPython
-
-
-`rpython/flowspace/`_              the FlowObjSpace_ implementing
-                                   `abstract interpretation`_
-
-`rpython/rlib/`_                   a `"standard library"`_ for RPython_
-                                   programs
-
-`rpython/rtyper/`_                 the `RPython Typer`_ 
-
-`rpython/rtyper/lltypesystem/`_    the `low-level type system`_ for
-                                   C-like backends
-
-`rpython/rtyper/ootypesystem/`_    the `object-oriented type system`_
-                                   for OO backends
-
-`rpython/memory/`_                 the `garbage collector`_ construction
-                                   framework
-
-`rpython/translator/`_             translation_ backends and support code
-
-`rpython/translator/backendopt/`_  general optimizations that run before a 
-                                   backend generates code
-
-`rpython/translator/c/`_           the `GenC backend`_, producing C code
-                                   from an
-                                   RPython program (generally via the rtyper_)
-
-`rpython/translator/cli/`_         the `CLI backend`_ for `.NET`_
-                                   (Microsoft CLR or Mono_)
-
-`pypy/goal/`_                      our `main PyPy-translation scripts`_
-