Commits

Lenard Lindstrom committed 41bc48c

Populating the project

Comments (0)

Files changed (7)

+A preliminary module tracker.
+
+This package tracks module imports and generates a report on program
+completion. It has been added to Pygame's SVN with the purpose of
+incorporating it into the unit tests. To that end it need testing
+and report feature replaced some kind of runtime inspection mechanism.
+Possibly the report generation can be optional. The data collection
+sets in reporter can also be exposed with an api. Finally, should
+it be a single module?
+
+See the playmus.py example for how it is currently used.

trackmod/__init__.py

+# package trackmod
+
+"""A package for tracking module use
+
+Exports start(repfilepth).
+
+"""
+
+from trackmod import reporter  # Want this first
+import sys
+import atexit
+
+from trackmod import importer
+
+try:
+    installed
+except NameError:
+    installed = False
+
+def generate_report(repfilepth):
+    try:
+        repfile = open(repfilepth, 'w')
+    except:
+        return
+    try:
+        reporter.write_report(repfile)
+    finally:
+        repfile.close()
+    
+def start(repfilepth):
+    global installed
+
+    if not installed:
+        sys.meta_path.insert(0, importer)
+        installed = True
+        atexit.register(generate_report, repfilepth)
+
+reporter.init()
+
+
+

trackmod/importer.py

+# module trackmod.importer
+
+"""A sys.meta_path importer for tracking module usage."""
+
+import sys
+
+from trackmod import loader
+from trackmod import module
+
+
+def find_module(fullname, path=None):
+    if fullname in sys.modules:
+        return None
+    try:
+        m = module.Module(fullname)
+        sys.modules[fullname] = m
+        try:
+            reload(m)
+        finally:
+            del sys.modules[fullname]
+    except ImportError:
+        return None
+    m.__class__ = module.TrackerModule
+    loader.add(fullname, m)
+    return loader
+
+

trackmod/keylock.py

+# module lazyimp.keylock
+
+"""Per-key reentrant thread locks"""
+
+import threading
+
+class Lock(object):
+    """A per-key reentrant thread lock
+
+    KeyLock(Key) => acquired lock
+
+    KeyLock provides locking by key. If others threads hold a non-freed Lock
+    instance for the given key then the constructor blocks until the other
+    threads have freed their instances.
+
+    """
+
+    cache = {}
+    cache_lock = threading.Lock()
+
+    class Error(Exception):
+        pass
+
+    def __new__(cls, key):
+        """Return a new acquired Lock instance"""
+        cls.cache_lock.acquire()
+        try:
+            try:
+                lock, count = cls.cache[key]
+            except KeyError:
+                lock = threading.RLock()
+                count = 0
+            cls.cache[key] = lock, count + 1
+        finally:
+            cls.cache_lock.release()
+        obj = object.__new__(cls)
+        obj.key = key
+        lock.acquire()
+        return obj
+
+    def free(self):
+        """Release the lock"""
+        self.cache_lock.acquire()
+        try:
+            try:
+                key = self.key
+            except AttributeError:
+                raise self.Error("Lock already freed")
+            lock, count = self.cache.pop(key)
+            lock.release()
+            if count > 1:
+                self.cache[key] = lock, count - 1
+            del self.key
+        finally:
+            self.cache_lock.release()
+
+    def __del__(self):
+        """Release the lock
+        
+        Yes, this method is here but should not be relied upon.
+        Call free explicitly.
+
+        """
+        try:
+            self.free()
+        except self.Error:
+            pass

trackmod/loader.py

+# module trackmod.loader
+
+"""A sys.meta_path loader for module usage tracking."""
+
+import sys
+import threading
+
+from trackmod import reporter
+
+module_table = {}
+module_table_lock = threading.Lock()
+
+def add(name, module):
+    module_table_lock.acquire()
+    try:
+        module_table[name] = module
+    finally:
+        module_table_lock.release()
+
+def pop(name):
+    module_table_lock.acquire()
+    try:
+        return module_table.pop(name)
+    finally:
+        module_table_lock.release()
+
+def load_module(fullname):
+    m = pop(fullname)
+    sys.modules[fullname] = m
+    reporter.add_loaded(fullname)
+    return m
+

trackmod/module.py

+# module trackmod.module
+
+"""Implements a module usage tracker module type"""
+
+from trackmod import keylock
+from trackmod import reporter
+
+
+ModuleType = type(reporter)
+
+class Module(ModuleType):
+    # A heap subtype of the module type.
+    #
+    # Allows __class__ to be changed. Otherwise it is just the same.
+    # To preserve the module docs this description is a comment.
+    
+    pass
+
+
+class TrackerModule(ModuleType):
+    def __getattribute__(self, attr):
+        if attr in ['__name__', '__path__']:
+            # The name attribute is the one attribute guaranteed to not trigger
+            # an import. __path__ is just noise in the reporting.
+            return ModuleType.__getattribute__(self, attr)
+        report(self, attr)
+        # At this point self's type has changed so getattr will not
+        # recursively call this method.
+        return getattr(self, attr)
+
+
+def report(module, attr):
+    name = module.__name__  # Safe: no recursive call on __name__.
+    lock = keylock.Lock(name)
+    try:
+        reporter.add_accessed(name, attr)
+        ModuleType.__setattr__(module, '__class__', Module)
+    finally:
+        lock.free()

trackmod/reporter.py

+def listmods():
+    return [n for n, m in sys.modules.iteritems() if m is not None]
+
+import sys
+already_loaded = listmods()
+import threading
+
+
+def print_(*args, **kwds):
+    stream = kwds.get('file', sys.stdout)
+    sep = kwds.get('sep', ' ')
+    end = kwds.get('end', '\n')
+
+    if args:
+        stream.write(sep.join([str(arg) for arg in args]))
+    if end:
+        stream.write(end)
+
+def process_accessed():
+    acc_names = dict(accessed)
+    for name, attr in accessed:
+        parts = name.split('.')
+        for i in range(1, len(parts)):
+            subname = '.'.join(parts[0:i])
+            if subname not in acc_names:
+                acc_names[subname] = parts[i]
+    return set(acc_names.iteritems())
+
+def write_report(repfile):
+    def rep(*args, **kwds):
+        print_(file=repfile, *args, **kwds)
+
+    accessed = process_accessed()
+    rep("=== module usage report ===")
+    rep("\n-- modules already imported (ignored) --")
+    already_loaded.sort()
+    for name in already_loaded:
+        rep(name)
+    rep("\n-- modules added by trackmod (ignored) --")
+    added_by_trackmod.sort()
+    for name in added_by_trackmod:
+        rep(name)
+    rep("\n-- modules imported but not accessed --")
+    acc = set([n for n, ignored in accessed])
+    unaccessed = list(loaded - acc)
+    unaccessed.sort()
+    for name in unaccessed:
+        rep(name)
+    rep("\n-- modules accessed --")
+    acc = list(accessed)
+    acc.sort()
+    for name, attr in acc:
+        rep(name, "(%s)" % attr)
+    rep("\n=== end of report ===")
+
+def init():
+    global already_loaded, loaded, accessed, data_lock, added_by_trackmod
+    added_by_trackmod = list(set(listmods()) - set(already_loaded))
+    loaded = set()
+    accessed = set()
+    data_lock = threading.Lock()
+
+def add_loaded(name):
+    data_lock.acquire()
+    try:
+        loaded.add(name)
+    finally:
+        data_lock.release()
+
+def add_accessed(name, attr):
+    data_lock.acquire()
+    try:
+        accessed.add((name, attr))
+    finally:
+        data_lock.release()
+
+
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.