modulegraph / modulegraph / find_modules.py

"""
modulegraph.find_modules - High-level module dependency finding interface
=========================================================================

History
........

Originally (loosely) based on code in py2exe's build_exe.py by Thomas Heller.
"""
from __future__ import absolute_import

import sys
import os
import imp
import warnings

import modulegraph.modulegraph as modulegraph
from modulegraph.modulegraph import Alias
from modulegraph.util import imp_find_module

__all__ = [
    'find_modules', 'parse_mf_results'
]

def get_implies():
    result = {
        # imports done from builtin modules in C code (untrackable by modulegraph)
        "_curses":      ["curses"],
        "posix":        ["resource"],
        "gc":           ["time"],
        "time":         ["_strptime"],
        "datetime":     ["time"],
        "MacOS":        ["macresource"],
        "cPickle":      ["copy_reg", "cStringIO"],
        "parser":       ["copy_reg"],
        "codecs":       ["encodings"],
        "cStringIO":    ["copy_reg"],
        "_sre":         ["copy", "string", "sre"],
        "zipimport":    ["zlib"],
        # mactoolboxglue can do a bunch more of these
        # that are far harder to predict, these should be tracked
        # manually for now.

        # this isn't C, but it uses __import__
        "anydbm":       ["dbhash", "gdbm", "dbm", "dumbdbm", "whichdb"],
        # package aliases
        "wxPython.wx":  Alias('wx'),
    }

    if sys.version_info[:2] >= (2, 5):
        result["_elementtree"] = ["pyexpat"]

        import xml.etree
        files = os.listdir(xml.etree.__path__[0])
        for fn in files:
            if fn.endswith('.py') and fn != "__init__.py":
                result["_elementtree"].append("xml.etree.%s"%(fn[:-3],))

    if sys.version_info[:2] >= (2, 6):
        result['future_builtins'] = ['itertools']

    return result

def parse_mf_results(mf):
    """
    Return two lists: the first one contains the python files in the graph,
    the second the C extensions.

    :param mf: a :class:`modulegraph.modulegraph.ModuleGraph` instance
    """
    #for name, imports in get_hidden_imports().items():
    #    if name in mf.modules.keys():
    #        for mod in imports:
    #            mf.import_hook(mod)

    # Retrieve modules from modulegraph
    py_files = []
    extensions = []

    for item in mf.flatten():
        # There may be __main__ modules (from mf.run_script), but
        # we don't need it in the zipfile we build.
        if item.identifier == "__main__":
            continue
        src = item.filename
        if src:
            suffix = os.path.splitext(src)[1]

            if suffix in PY_SUFFIXES:
                py_files.append(item)
            elif suffix in C_SUFFIXES:
                extensions.append(item)
            else:
                raise TypeError("Don't know how to handle '%s'" % repr(src))

    # sort on the file names, the output is nicer to read
    py_files.sort(key=lambda v: v.filename)
    extensions.sort(key=lambda v: v.filename)
    return py_files, extensions


def plat_prepare(includes, packages, excludes):
    # used by Python itself
    includes.update(["warnings", "unicodedata", "weakref"])

    if not sys.platform.startswith('irix'):
        excludes.update([
            'AL',
            'sgi',
        ])

    if not sys.platform in ('mac', 'darwin'):
        # XXX - this doesn't look nearly complete
        excludes.update([
            'Audio_mac',
            'Carbon.File',
            'Carbon.Folder',
            'Carbon.Folders',
            'EasyDialogs',
            'MacOS',
            'macfs',
            'macostools',
            'macpath',
        ])

    if not sys.platform == 'win32':
        # only win32
        excludes.update([
            'ntpath',
            'nturl2path',
            'win32api',
            'win32con',
            'win32event',
            'win32evtlogutil',
            'win32evtlog',
            'win32file',
            'win32gui',
            'win32pipe',
            'win32process',
            'win32security',
            'pywintypes',
            'winsound',
            'win32',
            '_winreg',
         ])

    if not sys.platform == 'riscos':
        excludes.update([
             'riscosenviron',
             'riscospath',
             'rourl2path',
          ])

    if not sys.platform == 'dos' or sys.platform.startswith('ms-dos'):
        excludes.update([
            'dos',
        ])

    if not sys.platform == 'os2emx':
        excludes.update([
            'os2emxpath'
        ])

    excludes.update(set(['posix', 'nt', 'os2', 'mac', 'ce', 'riscos']) - set(sys.builtin_module_names))

    try:
        imp_find_module('poll')
    except ImportError:
        excludes.update([
            'poll',
        ])

def find_needed_modules(mf=None, scripts=(), includes=(), packages=(), warn=warnings.warn):
    if mf is None:
        mf = modulegraph.ModuleGraph()
    # feed Modulefinder with everything, and return it.

    for path in scripts:
        mf.run_script(path)

    for mod in includes:
        try:
            if mod[-2:] == '.*':
                mf.import_hook(mod[:-2], None, ['*'])
            else:
                mf.import_hook(mod)
        except ImportError:
            warn("No module named %s"%(mod,))

    for f in packages:
        # If modulegraph has seen a reference to the package, then
        # we prefer to believe that (imp_find_module doesn't seem to locate
        # sub-packages)
        m = mf.findNode(f)
        if m is not None:
            path = m.packagepath[0]
        else:
            # Find path of package
            # TODO: use imp_find_module_or_importer
            try:
                path = imp_find_module(f, mf.path)[1]
            except ImportError:
                warn("No package named %s" % f)
                continue

        # walk the path to find subdirs containing __init__.py files
        # scan the results (directory of __init__.py files)
        # first trim the path (of the head package),
        # then convert directory name in package name,
        # finally push into modulegraph.
        # FIXME:
        # 1) Needs to be adjusted for namespace packages in python 3.3
        # 2) Code is fairly dodgy and needs better tests
        for (dirpath, dirnames, filenames) in os.walk(path):
            if '__init__.py' in filenames and dirpath.startswith(path):
                package = f + '.' + dirpath[len(path)+1:].replace(os.sep, '.')
                if package.endswith('.'):
                    package = package[:-1]
                m = mf.import_hook(package, None, ["*"])
            else:
                # Exclude subtrees that aren't packages
                dirnames[:] = []


    return mf

#
# resource constants
#
PY_SUFFIXES = ['.py', '.pyw', '.pyo', '.pyc']
C_SUFFIXES = [
    _triple[0] for _triple in imp.get_suffixes()
    if _triple[2] == imp.C_EXTENSION
]

#
# side-effects
#

def _replacePackages():
    REPLACEPACKAGES = {
        '_xmlplus':     'xml',
    }
    for k,v in REPLACEPACKAGES.items():
        modulegraph.replacePackage(k, v)

_replacePackages()

def find_modules(scripts=(), includes=(), packages=(), excludes=(), path=None, debug=0):
    """
    High-level interface, takes iterables for:
        scripts, includes, packages, excludes

    And returns a :class:`modulegraph.modulegraph.ModuleGraph` instance,
    python_files, and extensions

    python_files is a list of pure python dependencies as modulegraph.Module objects,
    extensions is a list of platform-specific C extension dependencies as modulegraph.Module objects
    """
    scripts = set(scripts)
    includes = set(includes)
    packages = set(packages)
    excludes = set(excludes)
    plat_prepare(includes, packages, excludes)
    mf = modulegraph.ModuleGraph(
        path=path,
        excludes=(excludes - includes),
        implies=get_implies(),
        debug=debug,
    )
    find_needed_modules(mf, scripts, includes, packages)
    return mf

def test():
    if '-g' in sys.argv[1:]:
        sys.argv.remove('-g')
        dograph = True
    else:
        dograph = False
    if '-x' in sys.argv[1:]:
        sys.argv.remove('-x')
        doxref = True
    else:
        doxref= False

    scripts = sys.argv[1:] or [__file__]
    mf = find_modules(scripts=scripts)
    if doxref:
        mf.create_xref()
    elif dograph:
        mf.graphreport()
    else:
        mf.report()

if __name__ == '__main__':
    test()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.