Commits

Lenard Lindstrom committed 0babc45

trackmod: some simplications and new check_test.py

Comments (0)

Files changed (6)

 The package has been successfully tried on a zipped package.
 
 
-See the playmus.py example for how it is currently used.
+See the playmus.py and check_test.py for usage examples.
+
 
 Log:
 
+rev 1745: - Made optional the recording of a submodule import as an
+            access on the containing package.
+          - Added preliminary check_test.py unit test checker.
+          - Simplified checkmod data collection in importer and module.
+            Replaced with some extra report generation overhead in reporter.
+
 rev 1743: - Added optional module pattern to limit which modules are monitored.
           - Added optional continuous attribute access recording.
 
+import sys
+import os
+import trackmod
+trackmod.begin(pattern=['pygame', 'pygame.*'],
+               continuous=True,
+               submodule_accesses=False)
+skip = set(['pygame.locals', 'pygame.constants',
+            'pygame.base', 'pygame.threads'])
+
+sys.path.append('.')
+os.chdir('test')
+test_file = sys.argv[1]
+del sys.argv[1]
+try:
+    execfile(test_file)
+finally:
+    trackmod.end()
+    print "=== Pygame package submodule accesses ==="
+    print
+    accesses = [(n, a) for n, a in trackmod.get_accesses().iteritems()
+                       if n not in skip]
+    accesses.sort(key=lambda t: t[0])
+    for name, attributes in accesses:
+        print "%s (%s)" % (name, ', '.join(attributes))

trackmod/__init__.py

         else:
             _write_report(repfile)
 
-def begin(repfile=None, pattern=None, continuous=False):
+def begin(repfile=None,
+          pattern=None,
+          continuous=False,
+          submodule_accesses=True):
     """Start collecting import and module access information
 
     repfile (default no file) is the destination for an
     recording should stop with the first access or be continuous. Set False
     to stop after the first access, True for continuous recording.
 
+    submodule_accesses (default True) indicates whether submodules imports
+    are to be included as an access on the containing package.
+
     """
     global installed, collecting
 
         collecting = True
     if continuous:
         module.set_report_mode('continuous')
-    importer.begin(pattern)
+    importer.begin(pattern, submodule_accesses)
 
 def end():
     global collecting

trackmod/importer.py

 """A sys.meta_path importer for tracking module usage."""
 
 import sys
-from trackmod import module, reporter, namereg
+from trackmod import module, namereg
 
 try:
     collect_data
 
 no_modules = []  # Contains nothing.
 modules_of_interest = no_modules
+add_submodule_accesses = True
 
 
 class Loader(object):
 
 def find_module(fullname, path=None):
     if fullname in modules_of_interest and fullname not in sys.modules:
-        # Put this first so the order of inserts follows the order of calls to
-        # find_module: package, subpackage, etc.
-        reporter.add_import(fullname)
-
-        # reload doesn't get any tracked TrackerModule attributes.
+        # reload doesn't "get" any tracked TrackerModule attributes.
         m = module.TrackerModule(fullname)
 
         # Add m to modules so reload works and to prevent infinite recursion.
             try:
                 reload(m)
             except ImportError, e:
-                reporter.remove_import(fullname)
                 return None;
         finally:
             del sys.modules[fullname]
 
         # Add parent package access.
-        parts = fullname.rsplit('.', 1)
-        if len(parts) == 2:
-            try:
-                pkg = sys.modules[parts[0]]
+        if add_submodule_accesses:
+            parts = fullname.rsplit('.', 1)
+            if len(parts) == 2:
                 try:
-                    getattr(pkg, parts[1])
-                except AttributeError:
+                    pkg = sys.modules[parts[0]]
+                except KeyError:
                     pass
-            except KeyError:
-                pass
+                else:
+                    try:
+                        getattr(pkg, parts[1])
+                    except AttributeError:
+                        pass
 
         return Loader(fullname, m)
     else:
     global modules_of_interest
     modules_of_interest = no_modules
 
-def begin(pattern=None):
-    global modules_of_interest, collect_data
+def begin(pattern=None, submodule_accesses=True):
+    global modules_of_interest, collect_data, add_submodule_accesses
     if pattern is None:
         pattern = ['*']
     modules_of_interest = namereg.NameRegistry(pattern)
+    add_submodule_accesses = submodule_accesses

trackmod/module.py

 
 """Implements a module usage tracker module type"""
 
-from trackmod import keylock
-from trackmod import reporter
+import threading
 
-try:
-    ModuleType
-except NameError:
-    pass
-else:
-    # reload; reload imported modules
-    reload(keylock)
-    reload(reporter)
 
-ModuleType = type(reporter)
+ModuleType = type(threading)
 getattribute = ModuleType.__getattribute__
+accesses = set()
+accesses_lock = threading.RLock()
 
 
 class Module(ModuleType):
 
 
 class TrackerModule(ModuleType):
+    # A heap subtype of the module type that tracks attribute gets.
+    #
+    # Allows __class__ to be changed. Otherwise it is just the same.
+    # To preserve the module docs this description is a comment.
+
+    # Attributes to ignore in reporting. The module name is the one
+    # attribute guarenteed to not be recorded. The class is used by
+    # the reporter. The path is just noise.
+    ignored_attributes = set(['__name__', '__class__', '__path__'])
+    
     def __getattribute__(self, attr):
-        if attr in ['__name__', '__path__']:
-            # The name attribute is the one attribute guaranteed to not trigger
-            # an import. __path__ is just noise in the reporting.
+        if attr in TrackerModule.ignored_attributes:
             return getattribute(self, attr)
         report(self, attr)
         return getattribute(self, attr)
 
 
-def report_oneshot(module, attr):
-    name = module.__name__  # Safe: no recursive call on __name__.
-    lock = keylock.Lock(name)
+def report_continuous(module, attr):
+    accesses_lock.acquire()
     try:
-        reporter.add_access(name, attr)
-        ModuleType.__setattr__(module, '__class__', Module)
+        # Safe: no recursive call on __name__ attribute.
+        accesses.add((module.__name__, attr))
     finally:
-        lock.free()
-
-def report_continuous(module, attr):
-    name = module.__name__  # Safe: no recursive call on __name__.
-    lock = keylock.Lock(name)
-    try:
-        reporter.add_access(name, attr)
-    finally:
-        lock.free()
+        accesses_lock.release()
 
 def report_quit(module, attr):
-    name = module.__name__  # Safe: no recursive call on __name__.
-    lock = keylock.Lock(name)
-    try:
-        ModuleType.__setattr__(module, '__class__', Module)
-    finally:
-        lock.free()
-    
+    module.__class__ = Module
+
+def report_oneshot(module, attr):
+    report_continuous(module, attr)
+    report_quit(module, attr)
+
+report = report_oneshot
+
+
 def set_report_mode(mode=None):
     """Set whether access checking is oneshot or continuous
 
     else:
         raise ValueError("Unknown mode %s" % mode)
 
-report = report_oneshot
+
+def get_accesses():
+    accesses_lock.acquire()
+    try:
+        return sorted(accesses)
+    finally:
+        accesses_lock.release()
+

trackmod/reporter.py

 previous_imports = listmods()  #  Keep this after sys but before other imports.
 import threading
 
+import module
+
+
 # This module is does not need explicit thread protection since all calls
 # to the data entry methods are made while the import lock is acquired.
 collect_data = True
 accesses = None
 failed_imports = None
 
+try:
+    next
+except NameError:
+    def next(iterator):
+        return iterator.next()
+
+class Largest(object):
+    """This object is always greater than any other non Largest object"""
+    def __lt__(self, other):
+        return False
+    def __le__(self, other):
+        return self == other
+    def __eq__(self, other):
+        return isinstance(other, Largest)
+    def __ne__(self, other):
+        not self == other
+    def __gt__(self, other):
+        return True
+    def __ge__(self, other):
+        return True
 
 def process_accessed():
     acc_names = dict(accessed)
 
 def get_imports():
     """Return a new sorted name list of imported modules"""
-    return sorted(accesses.iterkeys())
+    tracked_types = (module.Module, module.TrackerModule)
+    return sorted(n for n, m in list(sys.modules.iteritems())
+                    if isinstance(m, tracked_types))
 
 def get_unaccessed_modules():
     """Return a new sorted name list of unaccessed imported modules"""
-    return sorted(n for n, a in accesses.iteritems() if not a)
-    
+    unaccessed = []
+    iaccessed = iter(get_accessed_modules())
+    accessed_name = ''
+    for imports_name in get_imports():
+        while accessed_name < imports_name:
+            try:
+                accessed_name = next(iaccessed)
+            except StopIteration:
+                accessed_name = Largest()
+        if imports_name < accessed_name:
+            unaccessed.append(imports_name)
+    return unaccessed
+
 def get_accessed_modules():
     """Return a new sorted name list of accessed modules"""
-    return sorted(n for n, a in accesses.iteritems() if a)
+    accessed = []
+    previous_name = ''
+    for name, ignored in module.get_accesses():
+        if name != previous_name:
+            accessed.append(name)
+            previous_name = name
+    return accessed
 
 def get_accesses():
     """Return a new dictionary of sorted lists of attributes by module name"""
-    return dict((n, sorted(a)) for n, a in accesses.iteritems() if a)
+    accesses = {}
+    previous_name = ''
+    for name, attribute in module.get_accesses():
+        if name != previous_name:
+            attributes = []
+            accesses[name] = attributes
+            previous_name = name
+        attributes.append(attribute)
+    return accesses
 
 
 
 
+
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.