Commits

mattip  committed 93cbc90 Merge

merge default into branch

  • Participants
  • Parent commits f06471f, ee9cee4
  • Branches win32-stdlib

Comments (0)

Files changed (18)

File lib-python/2.7/pickle.py

             # else tmp is empty, and we're done
 
     def save_dict(self, obj):
-        modict_saver = self._pickle_moduledict(obj)
+        modict_saver = self._pickle_maybe_moduledict(obj)
         if modict_saver is not None:
             return self.save_reduce(*modict_saver)
 
                 write(SETITEM)
             # else tmp is empty, and we're done
 
-    def _pickle_moduledict(self, obj):
+    def _pickle_maybe_moduledict(self, obj):
         # save module dictionary as "getattr(module, '__dict__')"
+        try:
+            name = obj['__name__']
+            if type(name) is not str:
+                return None
+            themodule = sys.modules[name]
+            if type(themodule) is not ModuleType:
+                return None
+            if themodule.__dict__ is not obj:
+                return None
+        except (AttributeError, KeyError, TypeError):
+            return None
 
-        # build index of module dictionaries
-        try:
-            modict = self.module_dict_ids
-        except AttributeError:
-            modict = {}
-            from sys import modules
-            for mod in modules.values():
-                if isinstance(mod, ModuleType):
-                    modict[id(mod.__dict__)] = mod
-            self.module_dict_ids = modict
-
-        thisid = id(obj)
-        try:
-            themodule = modict[thisid]
-        except KeyError:
-            return None
-        from __builtin__ import getattr
         return getattr, (themodule, '__dict__')
 
 

File pypy/doc/coding-guide.rst

     >>>> cPickle.__file__
     '/home/hpk/pypy-dist/lib_pypy/cPickle..py'
 
-    >>>> import opcode
-    >>>> opcode.__file__
-    '/home/hpk/pypy-dist/lib-python/modified-2.7/opcode.py'
-
     >>>> import os
     >>>> os.__file__
     '/home/hpk/pypy-dist/lib-python/2.7/os.py'
 
     contains pure Python reimplementation of modules.
 
-*lib-python/modified-2.7/*
-
-    The files and tests that we have modified from the CPython library.
-
 *lib-python/2.7/*
 
-    The unmodified CPython library. **Never ever check anything in there**.
+    The modified CPython library.
 
 .. _`modify modules`:
 
 by default and CPython has a number of places where it relies
 on some classes being old-style.
 
-If you want to change a module or test contained in ``lib-python/2.7``
-then make sure that you copy the file to our ``lib-python/modified-2.7``
-directory first.  In mercurial commandline terms this reads::
-
-    $ hg cp lib-python/2.7/somemodule.py lib-python/modified-2.7/
-
-and subsequently you edit and commit
-``lib-python/modified-2.7/somemodule.py``.  This copying operation is
-important because it keeps the original CPython tree clean and makes it
-obvious what we had to change.
+We just maintain those changes in place,
+to see what is changed we have a branch called `vendot/stdlib`
+wich contains the unmodified cpython stdlib
 
 .. _`mixed module mechanism`:
 .. _`mixed modules`:

File pypy/doc/conf.py

 # built documents.
 #
 # The short X.Y version.
-version = '1.8'
+version = '1.9'
 # The full version, including alpha/beta/rc tags.
-release = '1.8'
+release = '1.9'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

File pypy/doc/cppyy.rst

 .. _`recent snapshot`: http://cern.ch/wlav/reflex-2012-05-02.tar.bz2
 .. _`gccxml`: http://www.gccxml.org
 
-Next, get the `PyPy sources`_, select the reflex-support branch, and build
-pypy-c.
+Next, get the `PyPy sources`_, select the reflex-support branch, and build.
 For the build to succeed, the ``$ROOTSYS`` environment variable must point to
-the location of your ROOT (or standalone Reflex) installation::
+the location of your ROOT (or standalone Reflex) installation, or the
+``root-config`` utility must be accessible through ``PATH`` (e.g. by adding
+``$ROOTSYS/bin`` to ``PATH``).
+In case of the former, include files are expected under ``$ROOTSYS/include``
+and libraries under ``$ROOTSYS/lib``.
+Then run the translation to build ``pypy-c``::
 
     $ hg clone https://bitbucket.org/pypy/pypy
     $ cd pypy
 code::
 
     $ genreflex MyClass.h
-    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so -L$ROOTSYS/lib -lReflex
 
 Now you're ready to use the bindings.
 Since the bindings are designed to look pythonistic, it should be
 That's all there is to it!
 
 
+Automatic class loader
+======================
+There is one big problem in the code above, that prevents its use in a (large
+scale) production setting: the explicit loading of the reflection library.
+Clearly, if explicit load statements such as these show up in code downstream
+from the ``MyClass`` package, then that prevents the ``MyClass`` author from
+repackaging or even simply renaming the dictionary library.
+
+The solution is to make use of an automatic class loader, so that downstream
+code never has to call ``load_reflection_info()`` directly.
+The class loader makes use of so-called rootmap files, which ``genreflex``
+can produce.
+These files contain the list of available C++ classes and specify the library
+that needs to be loaded for their use.
+By convention, the rootmap files should be located next to the reflection info
+libraries, so that they can be found through the normal shared library search
+path.
+They can be concatenated together, or consist of a single rootmap file per
+library.
+For example::
+
+    $ genreflex MyClass.h --rootmap=libMyClassDict.rootmap --rootmap-lib=libMyClassDict.so
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so -L$ROOTSYS/lib -lReflex
+
+where the first option (``--rootmap``) specifies the output file name, and the
+second option (``--rootmap-lib``) the name of the reflection library where
+``MyClass`` will live.
+It is necessary to provide that name explicitly, since it is only in the
+separate linking step where this name is fixed.
+If the second option is not given, the library is assumed to be libMyClass.so,
+a name that is derived from the name of the header file.
+
+With the rootmap file in place, the above example can be rerun without explicit
+loading of the reflection info library::
+
+    $ pypy-c
+    >>>> import cppyy
+    >>>> myinst = cppyy.gbl.MyClass(42)
+    >>>> print myinst.GetMyInt()
+    42
+    >>>> # etc. ...
+
+As a caveat, note that the class loader is currently limited to classes only.
+
+
 Advanced example
 ================
 The following snippet of C++ is very contrived, to allow showing that such
         std::string m_name;
     };
 
-    Base1* BaseFactory(const std::string& name, int i, double d) {
+    Base2* BaseFactory(const std::string& name, int i, double d) {
         return new Derived(name, i, d);
     }
 
 Now the reflection info can be generated and compiled::
 
     $ genreflex MyAdvanced.h --selection=MyAdvanced.xml
-    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so -L$ROOTSYS/lib -lReflex
 
 and subsequently be used from PyPy::
 
 
 A couple of things to note, though.
 If you look back at the C++ definition of the ``BaseFactory`` function,
-you will see that it declares the return type to be a ``Base1``, yet the
+you will see that it declares the return type to be a ``Base2``, yet the
 bindings return an object of the actual type ``Derived``?
 This choice is made for a couple of reasons.
 First, it makes method dispatching easier: if bound objects are always their
         int m_i;
     };
 
-    template class std::vector<MyClass>;
+    #ifdef __GCCXML__
+    template class std::vector<MyClass>;   // explicit instantiation
+    #endif
 
 If you know for certain that all symbols will be linked in from other sources,
 you can also declare the explicit template instantiation ``extern``.
 internal namespace, rather than in the iterator classes.
 One way to handle this, is to deal with this once in a macro, then reuse that
 macro for all ``vector`` classes.
-Thus, the header above needs this, instead of just the explicit instantiation
-of the ``vector<MyClass>``::
+Thus, the header above needs this (again protected with
+``#ifdef __GCCXML__``), instead of just the explicit instantiation of the
+``vector<MyClass>``::
 
     #define STLTYPES_EXPLICIT_INSTANTIATION_DECL(STLTYPE, TTYPE)                      \
     template class std::STLTYPE< TTYPE >;                                             \
     $ cat MyTemplate.xml
     <lcgdict>
         <class pattern="std::vector<*>" />
-        <class pattern="__gnu_cxx::__normal_iterator<*>" />
-        <class pattern="__gnu_cxx::new_allocator<*>" />
+        <class pattern="std::vector<*>::iterator" />
         <class pattern="std::_Vector_base<*>" />
         <class pattern="std::_Vector_base<*>::_Vector_impl" />
-        <class pattern="std::allocator<*>" />
         <function name="__gnu_cxx::operator=="/>
         <function name="__gnu_cxx::operator!="/>
 
 
 Run the normal ``genreflex`` and compilation steps::
 
-    $ genreflex MyTemplate.h --selection=MyTemplate.xm
-    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so
+    $ genreflex MyTemplate.h --selection=MyTemplate.xml
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so -L$ROOTSYS/lib -lReflex
 
 Note: this is a dirty corner that clearly could do with some automation,
 even if the macro already helps.
 There are a couple of minor differences between PyCintex and cppyy, most to do
 with naming.
 The one that you will run into directly, is that PyCintex uses a function
-called ``loadDictionary`` rather than ``load_reflection_info``.
+called ``loadDictionary`` rather than ``load_reflection_info`` (it has the
+same rootmap-based class loader functionality, though, making this point
+somewhat moot).
 The reason for this is that Reflex calls the shared libraries that contain
 reflection info "dictionaries."
 However, in python, the name `dictionary` already has a well-defined meaning,

File pypy/doc/cpython_differences.rst

   type and vice versa. For builtin types, a dictionary will be returned that
   cannot be changed (but still looks and behaves like a normal dictionary).
 
+* the ``__len__`` or ``__length_hint__`` special methods are sometimes
+  called by CPython to get a length estimate to preallocate internal arrays.
+  So far, PyPy never calls ``__len__`` for this purpose, and never calls
+  ``__length_hint__`` at all.
+
 
 .. include:: _ref.txt

File pypy/doc/getting-started-python.rst

 executable. The executable behaves mostly like a normal Python interpreter::
 
     $ ./pypy-c
-    Python 2.7.2 (0e28b379d8b3, Feb 09 2012, 19:41:03)
-    [PyPy 1.8.0 with GCC 4.4.3] on linux2
+    Python 2.7.2 (341e1e3821ff, Jun 07 2012, 15:40:31)
+    [PyPy 1.9.0 with GCC 4.4.3] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
-    And now for something completely different: ``this sentence is false''
+    And now for something completely different: ``RPython magically makes you rich
+    and famous (says so on the tin)''
+
     >>>> 46 - 4
     42
     >>>> from test import pystone
    ./include/
    ./lib_pypy/
    ./lib-python/2.7
-   ./lib-python/modified-2.7
    ./site-packages/
 
 The hierarchy shown above is relative to a PREFIX directory.  PREFIX is

File pypy/doc/getting-started.rst

 PyPy is ready to be executed as soon as you unpack the tarball or the zip
 file, with no need to install it in any specific location::
 
-    $ tar xf pypy-1.8-linux.tar.bz2
-    $ ./pypy-1.8/bin/pypy
-    Python 2.7.2 (0e28b379d8b3, Feb 09 2012, 19:41:03)
-    [PyPy 1.8.0 with GCC 4.4.3] on linux2
+    $ tar xf pypy-1.9-linux.tar.bz2
+    $ ./pypy-1.9/bin/pypy
+    Python 2.7.2 (341e1e3821ff, Jun 07 2012, 15:40:31)
+    [PyPy 1.9.0 with GCC 4.4.3] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
     And now for something completely different: ``it seems to me that once you
     settle on an execution / object model and / or bytecode format, you've already
 
     $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py
 
-    $ ./pypy-1.8/bin/pypy distribute_setup.py
+    $ ./pypy-1.9/bin/pypy distribute_setup.py
 
-    $ ./pypy-1.8/bin/pypy get-pip.py
+    $ ./pypy-1.9/bin/pypy get-pip.py
 
-    $ ./pypy-1.8/bin/pip install pygments  # for example
+    $ ./pypy-1.9/bin/pip install pygments  # for example
 
-3rd party libraries will be installed in ``pypy-1.8/site-packages``, and
-the scripts in ``pypy-1.8/bin``.
+3rd party libraries will be installed in ``pypy-1.9/site-packages``, and
+the scripts in ``pypy-1.9/bin``.
 
 Installing using virtualenv
 ---------------------------

File pypy/doc/index.rst

 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 1.8`_: the latest official release
+* `Release 1.9`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 1.8`: http://pypy.org/download.html
+.. _`Release 1.9`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html
 Windows, on top of .NET, and on top of Java.
 To dig into PyPy it is recommended to try out the current
 Mercurial default branch, which is always working or mostly working,
-instead of the latest release, which is `1.8`__.
+instead of the latest release, which is `1.9`__.
 
-.. __: release-1.8.0.html
+.. __: release-1.9.0.html
 
 PyPy is mainly developed on Linux and Mac OS X.  Windows is supported,
 but platform-specific bugs tend to take longer before we notice and fix

File pypy/doc/release-1.9.0.rst

+====================
+PyPy 1.9 - Yard Wolf
+====================
+
+We're pleased to announce the 1.9 release of PyPy. This release brings mostly
+bugfixes, performance improvements, other small improvements and overall
+progress on the `numpypy`_ effort.
+It also brings an improved situation on Windows and OS X.
+
+You can download the PyPy 1.9 release here:
+
+    http://pypy.org/download.html 
+
+.. _`numpypy`: http://pypy.org/numpydonate.html
+
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 1.9 and cpython 2.7.2`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64 or
+Windows 32.  Windows 64 work is still stalling, we would welcome a volunteer
+to handle that.
+
+.. _`pypy 1.9 and cpython 2.7.2`: http://speed.pypy.org
+
+
+Thanks to our donors
+====================
+
+But first of all, we would like to say thank you to all people who
+donated some money to one of our four calls:
+
+  * `NumPy in PyPy`_ (got so far $44502 out of $60000, 74%)
+
+  * `Py3k (Python 3)`_ (got so far $43563 out of $105000, 41%)
+
+  * `Software Transactional Memory`_ (got so far $21791 of $50400, 43%)
+
+  * as well as our general PyPy pot.
+
+Thank you all for proving that it is indeed possible for a small team of
+programmers to get funded like that, at least for some
+time.  We want to include this thank you in the present release
+announcement even though most of the work is not finished yet.  More
+precisely, neither Py3k nor STM are ready to make it in an official release
+yet: people interested in them need to grab and (attempt to) translate
+PyPy from the corresponding branches (respectively ``py3k`` and
+``stm-thread``).
+
+.. _`NumPy in PyPy`: http://pypy.org/numpydonate.html
+.. _`Py3k (Python 3)`: http://pypy.org/py3donate.html
+.. _`Software Transactional Memory`: http://pypy.org/tmdonate.html
+
+Highlights
+==========
+
+* This release still implements Python 2.7.2.
+
+* Many bugs were corrected for Windows 32 bit.  This includes new
+  functionality to test the validity of file descriptors; and
+  correct handling of the calling convensions for ctypes.  (Still not
+  much progress on Win64.) A lot of work on this has been done by Matti Picus
+  and Amaury Forgeot d'Arc.
+
+* Improvements in ``cpyext``, our emulator for CPython C extension modules.
+  For example PyOpenSSL should now work.  We thank various people for help.
+
+* Sets now have strategies just like dictionaries. This means for example
+  that a set containing only ints will be more compact (and faster).
+
+* A lot of progress on various aspects of ``numpypy``. See the `numpy-status`_
+  page for the automatic report.
+
+* It is now possible to create and manipulate C-like structures using the
+  PyPy-only ``_ffi`` module.  The advantage over using e.g. ``ctypes`` is that
+  ``_ffi`` is very JIT-friendly, and getting/setting of fields is translated
+  to few assembler instructions by the JIT. However, this is mostly intended
+  as a low-level backend to be used by more user-friendly FFI packages, and
+  the API might change in the future. Use it at your own risk.
+
+* The non-x86 backends for the JIT are progressing but are still not
+  merged (ARMv7 and PPC64).
+
+* JIT hooks for inspecting the created assembler code have been improved.
+  See `JIT hooks documentation`_ for details.
+
+* ``select.kqueue`` has been added (BSD).
+
+* Handling of keyword arguments has been drastically improved in the best-case
+  scenario: proxy functions which simply forwards ``*args`` and ``**kwargs``
+  to another function now performs much better with the JIT.
+
+* List comprehension has been improved.
+
+.. _`numpy-status`: http://buildbot.pypy.org/numpy-status/latest.html
+.. _`JIT hooks documentation`: http://doc.pypy.org/en/latest/jit-hooks.html
+
+JitViewer
+=========
+
+There is a corresponding 1.9 release of JitViewer which is guaranteed to work
+with PyPy 1.9. See the `JitViewer docs`_ for details.
+
+.. _`JitViewer docs`: http://bitbucket.org/pypy/jitviewer
+
+Cheers,
+The PyPy Team

File pypy/module/__pypy__/__init__.py

         'list_strategy'             : 'interp_magic.list_strategy',
         'validate_fd'               : 'interp_magic.validate_fd',
     }
+    if sys.platform == 'win32':
+        interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
 
     submodules = {
         "builders": BuildersModule,

File pypy/module/__pypy__/interp_magic.py

         rposix.validate_fd(fd)
     except OSError, e:
         raise wrap_oserror(space, e)
+
+def get_console_cp(space):
+    from pypy.rlib import rwin32    # Windows only
+    return space.newtuple([
+        space.wrap('cp%d' % rwin32.GetConsoleCP()),
+        space.wrap('cp%d' % rwin32.GetConsoleOutputCP()),
+        ])

File pypy/module/cpyext/test/test_cpyext.py

 class AppTestApi:
     def setup_class(cls):
         cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
-        from pypy.rlib.libffi import get_libc_name
+        from pypy.rlib.clibffi import get_libc_name
         cls.w_libc = cls.space.wrap(get_libc_name())
 
     def test_load_error(self):

File pypy/module/pypyjit/test_pypy_c/test__ffi.py

             #
             if os.name == 'nt':
                 from _ffi import WinDLL, types
-                libc = WinDLL(libc_name)
+                libc = WinDLL('Kernel32.dll')
                 sleep = libc.getfunc('Sleep', [types.uint], types.uint)
                 delays = [0]*n + [1000]
             else:

File pypy/rlib/rwin32.py

         'GetCurrentProcessId', [], DWORD)
     def GetCurrentProcessId():
         return rffi.cast(lltype.Signed, _GetCurrentProcessId())
+
+    _GetConsoleCP = winexternal('GetConsoleCP', [], DWORD)
+    _GetConsoleOutputCP = winexternal('GetConsoleOutputCP', [], DWORD)
+    def GetConsoleCP():
+        return rffi.cast(lltype.Signed, _GetConsoleCP())
+    def GetConsoleOutputCP():
+        return rffi.cast(lltype.Signed, _GetConsoleOutputCP())
+
     def os_kill(pid, sig):
         if sig == CTRL_C_EVENT or sig == CTRL_BREAK_EVENT:
             if GenerateConsoleCtrlEvent(sig, pid) == 0:

File pypy/rlib/test/test_rjvm.py

 import py
+py.test.skip('this is outdated. Check the jvm-improvements branch')
+
 try:
     import jpype
 except ImportError:

File pypy/test_all.py

 PyPy Test runner interface
 --------------------------
 
-Running test_all.py is equivalent to running py.test
-which you independently install, see
-http://pytest.org/getting-started.html
+Running pytest.py starts py.test, the testing tool
+we use in PyPy.  It is distributed along with PyPy,
+but you may get more information about it at
+http://pytest.org/.
+
+Note that it makes no sense to run all tests at once.
+You need to pick a particular subdirectory and run
+
+    cd pypy/.../test
+    ../../../pytest.py [options]
 
 For more information, use test_all.py -h.
 """

File pypy/translator/goal/app_main.py

             sys.path.append(dir)
             _seen[dir] = True
 
-def set_io_encoding(io_encoding):
+def set_io_encoding(io_encoding, io_encoding_output, errors, overridden):
     try:
         import _file
     except ImportError:
         set_file_encoding.argtypes = [ctypes.py_object, ctypes.c_char_p, ctypes.c_char_p]
     else:
         set_file_encoding = _file.set_file_encoding
-    if ":" in io_encoding:
-        encoding, errors = io_encoding.split(":", 1)
-    else:
-        encoding, errors = io_encoding, None
-    for f in [sys.stdin, sys.stdout, sys.stderr]:
-        set_file_encoding(f, encoding, errors)
+    for f, encoding in [(sys.stdin, io_encoding),
+                        (sys.stdout, io_encoding_output),
+                        (sys.stderr, io_encoding_output)]:
+        if isinstance(f, file) and (overridden or f.isatty()):
+            set_file_encoding(f, encoding, errors)
 
 # Order is significant!
 sys_flags = (
             print >> sys.stderr, "'import site' failed"
 
     readenv = not ignore_environment
-    io_encoding = ((readenv and os.getenv("PYTHONIOENCODING"))
-                   or sys.getfilesystemencoding())
+    io_encoding = readenv and os.getenv("PYTHONIOENCODING")
     if io_encoding:
-        set_io_encoding(io_encoding)
+        errors = None
+        if ":" in io_encoding:
+            io_encoding, errors = io_encoding.split(":", 1)
+        set_io_encoding(io_encoding, io_encoding, errors, True)
+    else:
+        if IS_WINDOWS:
+            import __pypy__
+            io_encoding, io_encoding_output = __pypy__.get_console_cp()
+        else:
+            io_encoding = io_encoding_output = sys.getfilesystemencoding()
+        if io_encoding:
+            set_io_encoding(io_encoding, io_encoding_output, None, False)
 
     pythonwarnings = readenv and os.getenv('PYTHONWARNINGS')
     if pythonwarnings:
 #!/usr/bin/env python
 """
-unit and functional testing with Python.
+PyPy Test runner interface
+--------------------------
+
+Running pytest.py starts py.test, the testing tool
+we use in PyPy.  It is distributed along with PyPy,
+but you may get more information about it at
+http://pytest.org/.
+
+Note that it makes no sense to run all tests at once.
+You need to pick a particular subdirectory and run
+
+    cd pypy/.../test
+    ../../../pytest.py [options]
+
+For more information, use pytest.py -h.
 """
 __all__ = ['main']
 
 from _pytest import __version__
 
 if __name__ == '__main__': # if run as a script or by 'python -m pytest'
+    import os
+    if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
+        print >> sys.stderr, __doc__
+        sys.exit(2)
+
     #XXX: sync to upstream later
     import pytest_cov
     raise SystemExit(main(plugins=[pytest_cov]))