Commits

Eric Snow committed 3878ceb Draft

consolidating from other projects; this is an intermediate state for the repo and the repo is presently low priority...

Comments (0)

Files changed (304)

+Copyright (c) 2012, Eric Snow
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+commonlib (cl): a namespace package holding commonly useful libraries
+
+Each subpackage in the namespace is separately installable from the PyPI, by
+name.  The whole namespace is also available there as a bundle under the
+name "commonlib".  All subpackages of the namespace share a single version.
+
+Subpackages:
+
+* cl.importing - help with the import system
+* cl.testing - testing help
+* cl.util - utilities that don't fit anywhere else
+
+Future Subpackages:
+
+* cl.data - a wide variety of data abstractions
+* cl.algorithms - tools for working with and implementations of various algo.
+* cl.py_backport - backports for libraries/features from newer Pythons
+* cl.recipes - recipes (a la ActiveState) that are candidates for promotion
+* cl.scripts - useful scripts
+# ...
+# ...
+# ...
+
+The core Package
+===================
+
+The core library is...
+
+For more information please refer to the contents of the Docs directory.
+
+
+Sub-packages
+===================
+
+Basic:
+
+  core (base)
+  core.importing
+  core.util
+  core.test
+  core.py_backports
+  core.py_custom
+
+Supplemental:
+
+  core.lib
+  core.localhost
+  core.scripts
+  core.experimental
+
+  core.clients
+  core.systems
+  core.modeling
+
+To see what is in each package, see the module (<?>.py) of the package.
+
+Each sub-package is available as a distribution package on PyPI.  The
+name of each is the same as above with an underscore replacing the dot
+(e.g. core.base => core_base).
+
+The "basic" sub-packages are also available as a bundled distribution
+under the name "core_basic".
+
+The basic and the supplemental packages are likewise all bundled in one
+distribution under the name "corelib".  The that is the main PyPI
+project for the core library.
+
+
+Repository Contents
+===================
+
+  INSTALL     - installation instructions
+  README      - this file
+  Docs/       - documentation and doc tools for the package
+  packages/   - the different sub-packages
+  setup.py    - the installer script
+  test_all.py - the script to run the test suite for all sub-packages
+
+
+References
+===================
+
+Main project page on PyPI:
+  http://pypi.python.org/.../corelib
+
+Documentation page on PyPI:
+  ...
+
+Repository and project site:
+  https://www.bitbucket.com/ericsnowcurrently/corelib
+

_unsorted/_backports/__init__.py

Empty file added.

_unsorted/_backports/importlib.py

+"""Backport of importlib.import_module from 3.x."""
+# While not critical (and in no way guaranteed!), it would be nice to keep this
+# code compatible with Python 2.3.
+import sys
+
+def _resolve_name(name, package, level):
+    """Return the absolute name of the module to be imported."""
+    if not hasattr(package, 'rindex'):
+        raise ValueError("'package' not set to a string")
+    dot = len(package)
+    for x in xrange(level, 1, -1):
+        try:
+            dot = package.rindex('.', 0, dot)
+        except ValueError:
+            raise ValueError("attempted relative import beyond top-level "
+                              "package")
+    return "%s.%s" % (package[:dot], name)
+
+
+def import_module(name, package=None):
+    """Import a module.
+
+    The 'package' argument is required when performing a relative import. It
+    specifies the package to use as the anchor point from which to resolve the
+    relative import to an absolute import.
+
+    """
+    if name.startswith('.'):
+        if not package:
+            raise TypeError("relative imports require the 'package' argument")
+        level = 0
+        for character in name:
+            if character != '.':
+                break
+            level += 1
+        name = _resolve_name(name[level:], package, level)
+    __import__(name)
+    return sys.modules[name]

_unsorted/_backports/ordereddict.py

+# Copyright (c) 2009 Raymond Hettinger
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+#     The above copyright notice and this permission notice shall be
+#     included in all copies or substantial portions of the Software.
+#
+#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#     OTHER DEALINGS IN THE SOFTWARE.
+
+# version 1.1 (2010-10-28)
+# http://pypi.python.org/pypi/ordereddict
+
+from UserDict import DictMixin
+
+class OrderedDict(dict, DictMixin):
+
+    def __init__(self, *args, **kwds):
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        try:
+            self.__end
+        except AttributeError:
+            self.clear()
+        self.update(*args, **kwds)
+
+    def clear(self):
+        self.__end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.__map = {}                 # key --> [key, prev, next]
+        dict.clear(self)
+
+    def __setitem__(self, key, value):
+        if key not in self:
+            end = self.__end
+            curr = end[1]
+            curr[2] = end[1] = self.__map[key] = [key, curr, end]
+        dict.__setitem__(self, key, value)
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        key, prev, next = self.__map.pop(key)
+        prev[2] = next
+        next[1] = prev
+
+    def __iter__(self):
+        end = self.__end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.__end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def popitem(self, last=True):
+        if not self:
+            raise KeyError('dictionary is empty')
+        if last:
+            key = reversed(self).next()
+        else:
+            key = iter(self).next()
+        value = self.pop(key)
+        return key, value
+
+    def __reduce__(self):
+        items = [[k, self[k]] for k in self]
+        tmp = self.__map, self.__end
+        del self.__map, self.__end
+        inst_dict = vars(self).copy()
+        self.__map, self.__end = tmp
+        if inst_dict:
+            return (self.__class__, (items,), inst_dict)
+        return self.__class__, (items,)
+
+    def keys(self):
+        return list(self)
+
+    setdefault = DictMixin.setdefault
+    update = DictMixin.update
+    pop = DictMixin.pop
+    values = DictMixin.values
+    items = DictMixin.items
+    iterkeys = DictMixin.iterkeys
+    itervalues = DictMixin.itervalues
+    iteritems = DictMixin.iteritems
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, self.items())
+
+    def copy(self):
+        return self.__class__(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        d = cls()
+        for key in iterable:
+            d[key] = value
+        return d
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedDict):
+            if len(self) != len(other):
+                return False
+            for p, q in  zip(self.items(), other.items()):
+                if p != q:
+                    return False
+            return True
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        return not self == other

_unsorted/_core_new/__init__.py

+"""core package
+
+The core package is a collection of cooperating subpackages.  Some are
+independent and others dependent.
+
+Distribution-wise, these are the distribution packages:
+
+--- basic:
+  * core_base
+     - ???
+  * core_importing
+     - engine
+     - module
+     - site
+  * core_util
+     - compact generic helpers
+  * core_py_backports
+  * core_py_custom
+  * core_modeling
+     - data (abstract)
+     - systems (abstract)
+        + common (lib)
+        + external_client
+        + master (incl. scripts)
+        + management (plugin)
+        + [...] (other roles)
+     - clients (abstract)
+
+--- more specific:
+  * core_lib
+  * core_localhost
+     - localhost
+     - maintenance
+     - logging
+     - access
+     - secure_cache
+  * core_scripts
+  * core_clients (concrete)
+  * core_systems (concrete)
+
+--- custom:
+  * [projects]
+  * [systems]
+  * [clients]
+
+
+The core packages are each set up to run as normal packages, with
+__init__.py, or as PEP 402-style packages using module extensions.
+
+"""
+

_unsorted/_core_new/_importing/engine.py

+"""core.importing.engine module
+
+much code borrowed liberally from:
+    https://bitbucket.org/jergosh/gsoc_import_engine/raw/default/Lib/importlib/engine.py
+see:
+    http://wiki.python.org/moin/SummerOfCode/PythonImportEnginePlanning
+
+stdlib modules that play a part here:
+  - imp
+  - importlib
+  - site
+  - sys
+  - pkgutil
+  - modulefinder
+  - runpy
+
+"""
+
+
+import imp
+from imp import (get_magic, get_tag,
+                 PY_SOURCE, PY_COMPILED, C_EXTENSION,
+                 PKG_DIRECTORY, C_BUILTIN, PY_FROZEN)
+
+try:
+    from threading import RLock
+except ImportError:
+    class RLock(object):
+        """A dummy RLock that acts like the import lock does."""
+        def acquire(self): return True
+        def release(self): pass
+        def _is_owned(self): return False
+
+from .module import Module
+
+
+class Site(object):
+    """A knock-off of the site module."""
+
+    def __init__(self, engine=None):
+
+        if engine is None:
+            engine = sysengine
+        self.engine = engine # where to find "sys.modules"
+
+        self.prefixes = []
+        self.usersiteenabled = True
+        self.usersite = None
+        self.userbase = None
+
+    @classmethod
+    def from_site(cls, site):
+        raise NotImplementedError
+
+    # XXX unnecessary?
+    def with_engine(self, engine):
+        """Return a context manager where the site uses that engine."""
+
+    def addsitedir(self, sitedir, known_paths=None):
+        raise NotImplementedError
+
+    def getsitepackages(self):
+        raise NotImplementedError
+
+    def getuserbase(self):
+        raise NotImplementedError
+
+    def getusersitepackages(self):
+        raise NotImplementedError
+
+
+class ImportLock(object):
+    """A wrapper around an RLock object.
+    
+    This is a replacement for the lock interface of the imp module.  It
+    is effectively a wrapper around a threading.RLock() instance, though
+    _is_owned() there is exposed as the locked property here.
+
+    ImportLock instances also support the context manager protocol.
+    
+    """
+    class __metaclass__(type):
+        """Out of sight, out of mind."""
+        def __init__(cls, n, b, ns): cls.locks = {}
+
+    def acquire(self):
+        locks = type(self).locks
+        if self not in locks:
+            locks[self] = RLock()
+        return locks[self].acquire()
+    def release(self):
+        locks = type(self).locks
+        if self not in locks:
+            locks[self] = RLock()
+        locks[self].release()
+    @property
+    def locked(self):
+        locks = type(self).locks
+        if self not in locks:
+            locks[self] = RLock()
+        return locks[self]._is_owned()
+
+    def __enter__(self):
+        self.acquire()
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.release()
+
+
+class ImportEngine(object):
+    """A container for the interpreter's import state.
+
+    The lock interface in the imp module is replaced by an actual lock
+    object.  It effectively wraps a threading.RLock() instance, though
+    _is_owned() there is exposed as 
+    
+
+    """
+
+    MODULE_CLASS = Module
+
+    def __init__(self, site=None):
+
+        if site is None:
+            site = syssite
+        self.modules = {}
+        self.meta_path = []
+        self.path_hooks = []
+        self.path_importer_cache = {}
+        self.path = []
+
+        self._lock = ImportLock()
+
+    @classmethod
+    def from_engine(cls, engine):
+        """Return a copy of the passed ImportEngine instance."""
+
+        engine = ImportEngine()
+
+        engine.modules = engine.modules[:]
+        engine.path = engine.path[:]
+        engine.path_hooks = engine.path_hooks[:]
+        engine.meta_path = engine.meta_path.copy()
+        engine.path_importer_cache = engine.path_importer_cache.copy()
+
+        return engine
+
+    ##############################
+    # start my additions
+
+    def as_importer(self):
+        """Return a PEP 302 importer that wraps the engine instance.
+
+        The only hangup is that the normal import machinery will check
+        sys.modules (interp->modules) for the module and a few other
+        things before kicking off the meta_path traversal.
+        
+        """
+        raise NotImplementedError
+
+    # XXX maybe instead allow for cls as an argument where needed
+    def with_module_class(self, cls):
+        """Return a context manager wherein cls gets used."""
+        raise NotImplementedError
+
+    @property
+    def lock(self):
+        return self._lock
+
+    # need different name?
+    def find_module(self, name, path=None):
+        raise NotImplementedError
+
+    # need different name?
+    def load_module(self, name, path=None, parent=None, attach=True):
+        raise NotImplementedError
+
+    def attach_module(self, module, ...):
+        raise NotImplementedError
+    
+    # copied from imp ############
+    # XXX bad idea?
+
+    def get_suffixes(self):
+        raise NotImplementedError
+
+    def find_module(self, name, path=None):
+        raise NotImplementedError
+
+    def load_module(self, name, modulefile):
+        raise NotImplementedError
+
+    def new_module(self, name):
+        return self.MODULE_CLASS(name, engine=self)
+
+    def reload(self, name):
+        raise NotImplementedError
+
+    def cache_from_source(self, path, debug_override=None):
+        raise NotImplementedError
+
+    def source_from_cache(self, path):
+        raise NotImplementedError
+
+    # end my additions
+    ##############################
+
+    def _gcd_import(self, name, package=None, level=0):
+        raise NotImplementedError
+
+    def import_module(self, name, package=None):
+        level = 0
+        if name.startswith('.'):
+            if not package:
+                raise TypeError("relative imports require the "
+                                "'package' argument")
+            for character in name:
+                if character != '.':
+                    break
+                level += 1
+        return self._gcd_import(name[level:], package, level)
+
+    def __import__(self, name, globals={}, locals={}, fromlist=[], level=0):
+        if not hasattr(name, 'rpartition'):
+            raise TypeError("module name must be str, "
+                            "not {}".format(type(name)))
+        if level == 0:
+            module = self._gcd_import(name)
+        else:
+            # __package__ is not guaranteed to be defined or could be
+            # set to None to represent that its proper value is unknown
+            package = globals.get('__package__')
+            if package is None:
+                package = globals['__name__']
+                if '__path__' not in globals:
+                    package = package.rpartition('.')[0]
+            module = self._gcd_import(name, package, level)
+        # The hell that is fromlist ...
+        if not fromlist:
+            # Return up to the first dot in 'name'. This is complicated
+            # by the fact that 'name' may be relative.
+            if level == 0:
+                return self.modules[name.partition('.')[0]]
+            elif not name:
+                return module
+            else:
+                cut_off = len(name) - len(name.partition('.')[0])
+                return self.modules[module.__name__[:-cut_off]]
+        else:
+            # If a package was imported, try to import stuff from fromlist.
+            if hasattr(module, '__path__'):
+                if '*' in fromlist and hasattr(module, '__all__'):
+                    fromlist = list(fromlist)
+                    fromlist.remove('*')
+                    fromlist.extend(module.__all__)
+                for x in (y for y in fromlist if not hasattr(module,y)):
+                    try:
+                        self._gcd_import('{0}.{1}'.format(module.__name__, x))
+                    except ImportError:
+                        pass
+            return module
+
+
+class GlobalSite(Site):
+
+    def __init__(self):
+        """Block Site's __init__."""
+
+    @property
+    def prefixes(self):
+        return site.PREFIXES
+    @property
+    def enableusersite(self):
+        return site.ENABLE_USER_SITE
+    @property
+    def usersite(self):
+        return site.USER_SITE
+    @property
+    def userbase(self):
+        return site.USER_BASE
+
+    def addsitedir(self, sitedir, known_paths=None):
+        site.addsitedir(sitedir, known_paths)
+    def getsitepackages(self):
+        site.getsitepackages()
+    def getuserbase(self):
+        site.getsitepackages()
+    def getusersitepackages(self):
+        site.getusersitepackages()
+
+
+class GlobalImportLock(ImportLock):
+    def acquire(self):
+        imp.acquire_lock()
+        return True
+    def release(self):
+        imp.release_lock()
+    @property
+    def locked(self):
+        return imp.lock_held()
+
+
+class GlobalImportEngine(ImportEngine):
+    """Wrapper around the global import state.
+
+    The attributes for instances of this engine are no more than
+    read-only properties wrapping the attributes on the global state.
+    
+    """
+
+    def __init__(self):
+        self._lock = GlobalImportLock()
+
+    @classmethod
+    def from_engine(cls, engine):
+        raise TypeError
+
+    @property
+    def modules(self):
+        return sys.modules
+
+    @property
+    def path(self):
+        return sys.path
+
+    @property
+    def path_hooks(self):
+        return sys.path_hooks
+
+    @property
+    def meta_path(self):
+        return sys.meta_path
+
+    @property
+    def path_importer_cache(self):
+        return sys.path_importer_cache
+
+sysengine = GlobalImportEngine()
+

_unsorted/_core_new/_importing/extensions.py

+"""core.extensions module
+
+At times it may not be provident to store all the contents of a single
+module in a single file, but creating a package sometimes has
+undesireable consequences.  The solution is to use module extensions.
+
+"module extension"
+
+A module extension is simply an ordinary Python module, stored in an
+ordinary ".py" file which should compile normally.  The module for which
+the extension exists is called the parent module.  The extension
+contains (and identifies in its "__all__") the names that are extended
+from the parent module.
+
+If module-level (import-time) execution has been extended out of the
+parent (instead of just names), then that executed code should be
+wrapped in a function in a module extension [file].  That function
+should then be called by the parent module during its own execution.
+
+"extension set"
+
+The combination of extensions for a parent is called an extension set.
+The extension set is identifed by a unique name, which may be the same
+name as the parent module, though there is no technical relationship
+between the two names.
+
+The names provided by each module extension in the set should be unique
+in the set.  As well, care should be taken to minimize the
+interdependence between the module extensions in the set.  Likewise any
+dependence on external modules that could cause the import of the parent
+module should be avoided, to prevent the risk of circular imports.
+
+"extension directory"
+
+The extension set for a parent module is stored in an extension
+directory that sits next to the parent module on the filessytem.  The
+name of this extension directory is a single underscore ("_"), followed
+by the name of the extension set.
+
+The extension directory need not be a package, since the extension
+mechanism does not use the normal import machinery.  So whether it is a
+package or not is irrelevant for the extension machinery.  However,
+relative imports in the individual module extensions cannot be used
+reliably because of this.
+
+When the parent module is imported, it calls the "pull_in_extension()"
+function or "pull_in_extension_set()".  This triggers the extension
+mechanism to pull the extensions and attach them to the parent module.
+At that point it's as though the extensions were defined in the parent
+module all along.
+
+Instead of the normal import mechanism, module extensions are loaded and
+executed directly.  Thus it does not matter if the extension directory
+is a package module or not.  In fact, "__init__" is not a valid
+extension name, since it may contribute to confusion about the
+relationship between package modules and extension directories.
+
+The name of each extension doesn't matter as long as it is a legal
+Python identifier (as with any Python module) and not "__init__".  There
+are two exceptions.
+
+First, by convention names that start with a single underscore should
+not be used for module extensions.  Instead, such names should only be
+used for files that hold logical sub-collections of actual module
+extensions.  The corresponding extension imports the sub-collection
+directly, effectively making it an extension for the extension.
+
+Second, one name is special: "__<name>__", where <name> is the name of
+the extension set.  The file with that name holds the main (or default)
+module extenion.  It is the principal location in the extension set to
+which the parent module extends itself.  As extensions become too large
+for this file they are then moved into their own files in the extension
+directory.
+
+Module extensions cannot be directories.  Any sub-directories in the
+extensions directory are ignored entirely.
+
+"""
+
+__all__ = (
+        "pull_in_extension",
+        "pull_in_extension_set",
+        )
+
+
+import sys
+import os
+
+from core import importing
+from core.py_custom.inspect import find_caller
+
+
+def pull_in_extension(name, path=None, names=None, extension=None):
+    """Import the requested extension to a parent module.
+
+    Parameters:
+
+      "name" is the name of the extensions set.
+      "path" is the directory that contains the extensions directory.
+      "names" are the names to pull from the module extension, if any.
+      "extension" is the name of the module extension to pull.
+
+    If "path" is not passed, it is extracted from the module of the
+    caller.
+    
+    If "extension" and "names" are both passed, the values for those
+    names in the extension module are returned, in the same order.
+    
+    If "extension" is passed and "names" is not passed, then the
+    extension module itself is returned.
+    
+    If "names" is passed and "extension" is not passed, the main
+    extension module is used.
+
+    Consequently, passing neither "names" nor "extension" causes the
+    main module extension to be returned. 
+
+    """
+
+    if path is None:
+        _globals = globals()
+        test = (lambda frame: frame.f_globals != _globals)
+        caller = imp.new_module(name)
+        caller.__dict__.update(find_caller(test).f_back.f_globals)
+        path = os.path.dirname(caller.__file__)
+
+    if module is None:
+        module = "__%s__" % name
+
+    modpath = os.path.join([path, "_%s" % name])
+    extension = importing.load_module(module, modpath, attach=False)
+    
+    if names is None:
+        return extension
+    return tuple(getattr(extension, name) for name in names)
+
+
+def pull_in_extension_set(name, parent, names=(), extensions=(), attach=False):
+    """Import the requested extensions to the parent module.
+
+    """
+
+    if isinstance(parent, str):
+        parent = sys.modules[parent]
+    path = os.path.dirname(parent.__file__)
+
+    # handle the names first
+    values = pull_in_extension(name, path, names, lazy=lazy)
+    for attr, value in zip(names, values):
+        setattr(parent, attr, value)
+
+    # then the submodules
+    for modname in submodules:
+        module = pull_in_extension(name, path, module=modname, lazy=lazy)
+        setattr(parent, modname, module)
+
+
+class Extension(importing.Module):
+    """A single module extension.
+
+    """
+
+    def __init__(self, name, path):
+        self.name = name
+        self.path = path
+
+    @classmethod
+    def create(cls, name, path):
+        ...
+
+
+class ExtensionSet(importing.ModuleSet):
+    """The set of extensions for a given module.
+    
+    This seems similar to sys.modules: a container for modules.
+
+    """
+
+    def __init__(self, name, parent=None, path=None):
+        self.name = name
+        self.parent = parent
+        self.path = path
+
+        self.extensions = {}
+
+    def pull_extension(self, name=None, names=None):
+        ...
+
+    def bind_extension(self, name=None, names=None):
+        ...
+
+    def bind_all(self, target=None, names=(), exts=(), attach=False):
+        ...
+

_unsorted/_core_new/_importing/hooks/import_engine.py

Empty file added.

_unsorted/_core_new/_importing/module.py

+"""core.importing module
+
+"""
+
+__all__ = (
+        "Module",
+        "LazyModule", "SmartModule", "ExtensionModule",
+        )
+
+
+import sys
+import imp
+
+
+ModuleType = type(sys)
+
+
+#################################################
+# module classes
+
+ModuleDescription = namedtuple("ModuleDescription", "suffix mode type")
+ModuleFile = namedtuple("ModuleFile", "file pathname description")
+
+
+class ModuleMeta(type):
+
+    def from_path(cls, path, name=None, **kwargs):
+        """Factory function for cls which pulls from the path."""
+        raise NotImplementedError
+
+    def from_module(cls, module, name=None, **kwargs):
+        """Factory function for cls which copies the module."""
+        if not hasattr(module, "items"):
+            module = getattr(module, self.MODULE_NAMESPACE_ATTR)
+        return cls.from_namespace(module, name, **kwargs)
+
+    def from_namespace(cls, namespace, name=None, **kwargs):
+        """Factory function for cls which copies the namespace."""
+        namespace = namespace.copy()
+        if "__builtins__" in namespace:
+            del namespace["__builtins__"]
+        namespace.update(kwargs)
+        if name is None:
+            name = namespace.get("__name__")
+        else:
+            namespace["__name__"] = name
+        module = cls(name)
+        module.__dict__.update(namespace)
+        return module
+
+    def find(cls, module):
+        """Find this module's ModuleFile."""
+        raise NotImplementedError
+
+    def build(cls, module):
+        """Execute the module and populate its namespace."""
+        raise NotImplementedError
+
+    def attach(cls, module, ...):
+        """Bind the module to its package and sys.modules."""
+        raise NotImplementedError
+
+    def reload(cls, module):
+        """Reload that module object."""
+        raise NotImplementedError
+
+
+#class Module(ModuleType):
+class Module(object):
+    """A straight-up replacement for the built-in module type.
+
+    By default Module objects will always have the following attributes:
+    
+      __doc__
+      __name__
+
+    Where __name__ is set to the passed name and __doc__ to None.  Here
+    are the other attributes that may be found on Module objects:
+
+      __all__       - a list of the module's "public" attribute names.
+      __builtins__  - the builtins used during execution for non-builtin
+                    modules (None or not set for "built-in" modules).
+                    For this class __builtins__ is explicitly not set.
+      __cached__    - the cached file (see PEP 3147).
+      __file__      - the path to the file from which the module was
+                    loaded (None or not set for "built-in" modules).
+      __loader__    - the loader that loaded the module (see PEP 302).
+      __package__   - the package to which the module belongs (see
+                    PEP 366).
+      __path__      - for packages, the list of paths to use when
+                    importing modules in the package (see PEP 302).
+
+    For more on these attributes, see the following resources (all at
+    <http://docs.python.org/dev>):
+
+      /reference/simple_stmts.html#the-import-statement
+      /library/importlib.html#importlib.abc.Loader.load_module
+      /library/runpy.html#runpy.run_module
+      /library/imp.html#imp.source_from_cache
+      /whatsnew/3.2.html#pep-3147-pyc-repository-directories
+      /tutorial/modules.html#packages-in-multiple-directories
+
+    PEPS:
+
+      302  - http://www.python.org/dev/peps/pep-0302/
+      366  - http://www.python.org/dev/peps/pep-0366/
+      3147 - http://www.python.org/dev/peps/pep-3147/
+      
+    When a module's code object is executed, it should be done using
+    the modules __dict__ as the globals passed in to exec().  Thus, any
+    module attributes must be set before module execution if they are to
+    be available during execution.
+    
+    Note: a package is just a module with __path__ set.  The package's
+    __init__.py is the actual code executed for that module.  When
+    submodules are imported, they are subsequently bound by their base
+    name to the namespace of the package and __package__ of the
+    submodule is set to the package.  A submodule can be a package
+    itself.
+
+    """
+    __metaclass__ = ModuleMeta
+
+    MODULE_NAMESPACE_ATTR = "__dict__"
+    MODULE_ATTRS = (
+            "__name__", "__doc__", "__file__",
+            "__loader__", "__all__", # optional
+            "__package__", "__loader__", # future
+            "__builtins__", # implementation detail
+            )
+    PACKAGE_ATTRS = ("__path__",)
+    
+    def __new__(cls, name, *args, **kwargs):
+        if not name or not isinstance(name, str):
+            raise TypeError("name must be a non-empty string")
+        return super(cls, cls).__new__(cls, name, *args, **kwargs)
+
+    def __init__(self, name, engine):
+        self.__name__ = name
+        self.__doc__ = None
+        self.__engine__ = engine
+
+    @property
+    def __module__(self):
+        return self
+
+    @property
+    def __builtins__(self):
+        raise TypeError("__builtins__ not available")
+
+
+class LazyModule(Module):
+    """Effectively a placeholder module.
+    
+    Any attribute access forces the lazy load and rebinds the original
+    name to the wrapped module (before anything else happens).
+
+    """
+
+    def __init__(self, name, wrapped):
+        type(self)._setattr(self, "__name__", name)
+        type(self)._setattr(self, "__doc__", None)
+        type(self)._setattr(self, "__wrapped__", wrapped)
+        type(self)._setattr(self, "_done", False)
+
+    # the triggers
+    def __getattribute__(self, name):
+        if name in ("__name__",):
+            return object.__getattribute__(self, name)
+        module = type(self).rebind(self)
+        return getattr(module, name)
+    def __setattr__(self, name, value):
+        module = type(self).rebind(self)
+        setattr(module, name, value)
+    def __delattr__(self, name):
+        module = type(self).rebind(self)
+        delattr(module, name)
+
+    # access work-around
+    def _getattr(self, name):
+        return object.__getattribute__(self, name)
+    def _setattr(self, name, value):
+        object.__setattr__(self, name, value)
+
+    # the meat of the class
+    def rebind(self, name=None):
+        """Force the wrapped module to be put into place."""
+
+        if type(self)._getattr(self, "_done"):
+            raise TypeError("can't rebind a module that has been rebound")
+        if name is None:
+            name = self.__name__
+        wrapped = type(self)._getattr(self, "__wrapped__")
+        if isinstance(wrapped, str):
+            wrapped = load_module(wrapped)
+            type(self)._setattr(self, "__wrapped__", wrapped)
+
+        attach_module(wrapped, name, verify=self)
+        rebind_module(wrapped, name, verify=self) 
+
+        type(self)._setattr(self, "_done", True)
+        return wrappped
+
+
+class SmartModule(Module):
+    """Some extra features added on to the base module.
+
+    The SmartModule methods are exposed as staticmethods on the
+    metaclass so that they do not interfere with the namespace of
+    module instances.
+
+    The exception to this rule is the __wrapped__ attribute, which
+    is bound to the module this one replaced, or to None.
+
+    """
+
+    class __metaclass__(type):
+        def from_module(cls, module, lazy=False, attach=False):
+            """Make a new SmartModule out of the passed module."""
+            if isinstance(module, str):
+                name = module
+            else:
+                name = module.__name__
+
+            newmodule = cls(name)
+            cls.assume(newmodule, module, lazy)
+            cls.attach(newmodule, name)
+            return newmodule
+
+        @staticmethod
+        def assume(module, wrapped, lazy=False):
+            if isinstance(module, str):
+                module = load_module(module)
+
+    def __init__(self, name):
+        super(type(self), self).__init__(name)
+        self.__wrapped__ = None
+
+    def assume(self, module):
+        """Take over the place of the passed module in sys.modules."""
+        self.__wrapped__ = module
+        ...
+
+    def find(self):
+        ...
+
+    def load(self):
+        ...
+
+    def attach(self):
+        ...
+
+
+class ExtendedModule(ModuleType):
+
+    BORROWED = ("__name__", "__doc__",
+                "__file__", "__path__", "__package__",
+                )
+
+    def __init__(self, original=None, lazy=False):
+
+        if not lazy:
+            original = self.load_module(original)
+        else:
+            find_module(original)
+
+        self.__wrapped__ = original
+        self.lazy = lazy
+
+    
+    @classmethod
+    def from_module(cls, module, lazy=False, replace=True):
+    #def from_module(cls, module, *, lazy=False, replace=True):
+        newmodule = cls(module, lazy)
+        if replace:
+            sys.modules[newmodule.__name__]
+
+
+#######################################
+# tests
+# 
+# ExtendedModule
+# * <result of ExtendedModule(original)>.__name__ == original.__name__
+# * <result of from_module(module)>.__name__ == module.__name__
+# * 
+# * 
+# * 
+# * 
+#
+# * 
+# 

_unsorted/_core_new/_importing/modulehacker.py

+## {{{ http://code.activestate.com/recipes/577740/ (r3)
+"""modulehacker module"""
+
+import sys
+import importlib
+
+_hackers = []
+def register(obj):
+    _hackers.append(obj)
+
+class Hacker:
+    def hack(self, module):
+        return module
+
+class Loader:
+    def __init__(self):
+        self.module = None
+    
+    def find_module(self, name, path):
+        sys.meta_path.remove(self)
+        self.module = importlib.import_module(name)
+        sys.meta_path.insert(0, self)
+        return self
+    
+    def load_module(self, name):
+        if not self.module:
+            raise ImportError("Unable to load module.")
+        module = self.module
+        for hacker in _hackers:
+            module = hacker.hack(module)
+        sys.modules[name] = module
+        return module
+
+sys.meta_path.insert(0, Loader())
+## end of http://code.activestate.com/recipes/577740/ }}}
+    

_unsorted/_core_new/_importing/pep382.py

+
+
+class PEP382Finder:
+    
+    def find_module(self, fullname, path=None):
+        if parent_is_ns_package(fullname):
+            ...
+
+        return None
+
+
+class PEP382Loader:
+    def __init__(self, ...):
+        ...
+
+    def load_module(self, fullname):
+        ...
+
+
+def activate():
+    sys.meta_path.append(PEP382Finder())
+
+

_unsorted/_core_new/_importing/pep402.py

Empty file added.

_unsorted/_core_new/_importing/tracker.py

+"""import_tracker module
+
+ImportTracker is the main type.
+
+TrackerStream provides logging for ImportTracker objects.
+
+"""
+
+__all__ = ("ImportTracker", "TrackerStream")
+
+
+import builtins
+import sys
+import time
+import pkgutil
+import importlib
+from inspect import currentframe, getframeinfo
+from collections import namedtuple
+
+builtins.__import__ = importlib.__import__
+
+class _Modules(dict):
+    """A singleton dict-like type for use as sys.modules."""
+
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls, *args, **kwargs)
+        return cls._instance
+    
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.trackers = []
+
+    def __getitem__(self, key):
+        frame = currentframe().f_back
+        if frame.f_globals["__name__"] == __name__:
+            imported = true
+        else:
+            context = getframeinfo(frame).code_context
+            imported = context and "import" in context
+        for tracker in self.trackers:
+            tracker._handle_get(key, imported)
+        return super().__getitem__(key)
+    
+    def __setitem__(self, key, value):
+        frame = currentframe().f_back
+        if frame.f_globals["__name__"] == __name__:
+            imported = True
+        else:
+            context = getframeinfo(frame).code_context
+            imported = context and "import" in context
+        for tracker in self.trackers:
+            tracker._handle_set(key, imported)
+        super().__setitem__(key, value)
+    
+    def __delitem__(self, key):
+        for tracker in self.trackers:
+            tracker._handle_del(key)
+        return super().__delitem__(key)
+        
+
+class _Importer:
+    """A PEP 302 loader that tracks the success of an import."""
+
+    def __init__(self, loader, tracker):
+        self.loader = loader
+        self.tracker = tracker
+    
+    def load_module(self, name):
+        touchtype = "failed"
+        try:
+            module = self.loader.load_module(name)
+            if name in sys.modules:
+                touchtype = "loaded"
+            elif module:
+                sys.modules[name] = module
+        finally:
+            self.tracker._touch(name, touchtype)
+
+_Event = namedtuple("_Event", "time name type")
+
+
+class TrackerStream:
+    """A wrapper around a logger with a StreamHandler.
+    
+    This is also a context manager in which the stream is activated on
+    its tracker:
+
+    >>> tracker = ImportTracker()
+    >>> with TrackerStream(open("/tmp/spam", "w"), tracker):
+    ...     import some_module
+    >>>
+
+    """
+
+    def __init__(self, stream, tracker):
+        self.tracker = tracker
+        self.tracker._streams.append(self)
+        self.stream = stream
+        self.logger = logging.getLogger(id(self))
+        self.logger.addHandler(logging.StreamHandler(stream))
+    
+    def __getattr__(self, name):
+        return getattr(self.logger, name)
+    
+    def __enter__(self):
+        self.tracker.__enter__()
+        if hasattr(self.stream, "__enter__"):
+            self.stream.__enter__()
+    
+    def __exit__(self, *args, **kwargs):
+        self.logger.close()
+        self.tracker._streams.remove(self)
+        if hasattr(self.stream, "__enter__"):
+            self.stream.__exit__(*args, **kwargs)
+        self.tracker.__exit__(*args, **kwargs)
+
+
+class ImportTracker:
+    """The main tracking class.
+
+    The key attributes are:
+      all - the full import history (in order)
+      touches - the categorized bits of the history
+
+    >>> tracker = ImportTracker()
+    >>> tracker.enable()
+    
+    >>> import some_module
+    >>> len(tracker.all)
+
+    """
+
+    def __init__(self, stream=None):
+        self._streams = []
+        if stream:
+            self._streams.append(TrackerStream(stream, self))
+
+        self._count = 0
+
+        self.all = []
+        self.touches = {}
+
+    def using(self, stream):
+        """Pulls out a stream from the tracker."""
+        return TrackerStream(stream, self)
+
+    def enable(self):
+        """Adds the tracker to the custom sys.meta_path."""
+        self._count += 1
+        if self._count == 1:
+            sys.meta_path.insert(0, self)
+            sys.modules = _Modules(sys.modules)
+            sys.modules.trackers.append(self)
+    def disable(self):
+        """Removes the tracker from sys.meta_path."""
+        self._count -= 1
+        if self._count == 0:
+            sys.meta_path.remove(self)
+            sys.modules.trackers.remove(self)
+            if not sys.modules.trackers:
+                sys.modules = dict(sys.modules)
+
+    def _touch(self, name, ttype):
+        event = _Event(time.time(), name, ttype)
+        self.all.append(event)
+        if ttype not in self.touches:
+            self.touches[ttype] = []
+        self.touches[ttype].append(event)
+
+        for stream in self._streams:
+            stream.info(event)
+
+    def _handle_get(self, name, imported=True):
+        if imported:
+            self._touch(name, "already imported")
+        else:
+            self._touch(name, "direct get")
+
+    def _handle_set(self, name, imported=True):
+        if imported:
+            self._touch(name, "newly imported")
+        else:
+            self._touch(name, "direct set")
+
+    def _handle_del(self, name):
+        self._touch(name, "direct del")
+
+    def __enter__(self):
+        self.enable()
+    def __exit__(self, *args, **kwargs):
+        self.disable()
+
+    def find_module(self, name, path=None):
+        meta_path = sys.meta_path[:]
+        meta_path.remove(self)
+        touchtype = "missing"
+        try:
+            for finder in meta_path:
+                loader = finder.find_module(name, path)
+                if loader is not None:
+                    break
+            else:
+                loader = pkgutil.ImpImporter().find_module(name, path)
+                if loader is None:
+                    return None
+            touchtype = "found"
+        finally:
+            self._touch(name, touchtype)
+        return _Importer(loader, self)
+
+
+if __name__ == "__main__":
+    tracker = ImportTracker()
+    tracker.enable()
+    try:
+        import x
+    except ImportError:
+        pass
+    import logging.handlers
+

_unsorted/_core_new/_py_backports/_argparse.py

+# Author: Steven J. Bethard <steven.bethard@gmail.com>.
+
+"""Command-line parsing library
+
+This module is an optparse-inspired command-line parsing library that:
+
+    - handles both optional and positional arguments
+    - produces highly informative usage messages
+    - supports parsers that dispatch to sub-parsers
+
+The following is a simple usage example that sums integers from the
+command-line and writes the result to a file::
+
+    parser = argparse.ArgumentParser(
+        description='sum the integers at the command line')
+    parser.add_argument(
+        'integers', metavar='int', nargs='+', type=int,
+        help='an integer to be summed')
+    parser.add_argument(
+        '--log', default=sys.stdout, type=argparse.FileType('w'),
+        help='the file where the sum should be written')
+    args = parser.parse_args()
+    args.log.write('%s' % sum(args.integers))
+    args.log.close()
+
+The module contains the following public classes:
+
+    - ArgumentParser -- The main entry point for command-line parsing. As the
+        example above shows, the add_argument() method is used to populate
+        the parser with actions for optional and positional arguments. Then
+        the parse_args() method is invoked to convert the args at the
+        command-line into an object with attributes.
+
+    - ArgumentError -- The exception raised by ArgumentParser objects when
+        there are errors with the parser's actions. Errors raised while
+        parsing the command-line are caught by ArgumentParser and emitted
+        as command-line messages.
+
+    - FileType -- A factory for defining types of files to be created. As the
+        example above shows, instances of FileType are typically passed as
+        the type= argument of add_argument() calls.
+
+    - Action -- The base class for parser actions. Typically actions are
+        selected by passing strings like 'store_true' or 'append_const' to
+        the action= argument of add_argument(). However, for greater
+        customization of ArgumentParser actions, subclasses of Action may
+        be defined and passed as the action= argument.
+
+    - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter,
+        ArgumentDefaultsHelpFormatter -- Formatter classes which
+        may be passed as the formatter_class= argument to the
+        ArgumentParser constructor. HelpFormatter is the default,
+        RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser
+        not to change the formatting for help text, and
+        ArgumentDefaultsHelpFormatter adds information about argument defaults
+        to the help.
+
+All other classes in this module are considered implementation details.
+(Also note that HelpFormatter and RawDescriptionHelpFormatter are only
+considered public as object names -- the API of the formatter objects is
+still considered an implementation detail.)
+"""
+
+__version__ = '1.1'
+__all__ = [
+    'ArgumentParser',
+    'ArgumentError',
+    'ArgumentTypeError',
+    'FileType',
+    'HelpFormatter',
+    'ArgumentDefaultsHelpFormatter',
+    'RawDescriptionHelpFormatter',
+    'RawTextHelpFormatter',
+    'Namespace',
+    'Action',
+    'ONE_OR_MORE',
+    'OPTIONAL',
+    'PARSER',
+    'REMAINDER',
+    'SUPPRESS',
+    'ZERO_OR_MORE',
+]
+
+
+import collections as _collections
+import copy as _copy
+import os as _os
+import re as _re
+import sys as _sys
+import textwrap as _textwrap
+
+from gettext import gettext as _
+
+
+def _callable(obj):
+    return hasattr(obj, '__call__') or hasattr(obj, '__bases__')
+
+
+SUPPRESS = '==SUPPRESS=='
+
+OPTIONAL = '?'
+ZERO_OR_MORE = '*'
+ONE_OR_MORE = '+'
+PARSER = 'A...'
+REMAINDER = '...'
+_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args'
+
+# =============================
+# Utility functions and classes
+# =============================
+
+class _AttributeHolder(object):
+    """Abstract base class that provides __repr__.
+
+    The __repr__ method returns a string in the format::
+        ClassName(attr=name, attr=name, ...)
+    The attributes are determined either by a class-level attribute,
+    '_kwarg_names', or by inspecting the instance __dict__.
+    """
+
+    def __repr__(self):
+        type_name = type(self).__name__
+        arg_strings = []
+        for arg in self._get_args():
+            arg_strings.append(repr(arg))
+        for name, value in self._get_kwargs():
+            arg_strings.append('%s=%r' % (name, value))
+        return '%s(%s)' % (type_name, ', '.join(arg_strings))
+
+    def _get_kwargs(self):
+        return sorted(self.__dict__.items())
+
+    def _get_args(self):
+        return []
+
+
+def _ensure_value(namespace, name, value):
+    if getattr(namespace, name, None) is None:
+        setattr(namespace, name, value)
+    return getattr(namespace, name)
+
+
+# ===============
+# Formatting Help
+# ===============
+
+class HelpFormatter(object):