Commits

akalias  committed 7e9abc5

Added memoized get_tags class/function for inheritable __tags__, won't redirect_output on 'interactive' tagged tests, run_tests.py will run tests prefixed with todo_, gen_stubs will find already_tested todo_ tests also handles test_1_xxxxx name so can order interactive tests (ala cdrom_test.py)

  • Participants
  • Parent commits 9dbb2bc

Comments (0)

Files changed (6)

File run_tests.py

 import sys, os, re, subprocess, time, optparse
 import pygame.threads, pygame
 
-from test_runner import *
+from test_runner import prepare_test_env, run_test, combine_results, \
+                        test_failures, get_test_results, from_namespace, \
+                        TEST_RESULTS_START
+
 from pprint import pformat
 
 main_dir, test_subdir, fake_test_subdir = prepare_test_env()
         m.endswith('_test') and m or ('%s_test' % m) for m in args
     ]
 else:
-    if options.subprocess: ignore = SUBPROCESS_IGNORE.copy()
-    else: ignore = IGNORE.copy()
+    if options.subprocess: ignore = SUBPROCESS_IGNORE
+    else: ignore = IGNORE
 
     # TODO: add option to run only INTERACTIVE, or include them, etc
-    ignore |= INTERACTIVE
+    ignore = ignore | INTERACTIVE
 
     test_modules = []
     for f in sorted(os.listdir(test_subdir)):

File test/cdrom_test.py

 #################################### IMPORTS ###################################
 
+__tags__ = ['interactive']
+
 import test_utils
 import test.unittest as unittest
-from test_utils import test_not_implemented
+from test_utils import test_not_implemented, question, prompt
 
 import pygame
 
-def question(q):
-    return raw_input('%s ' % q).lower().strip() == 'y'
-
-def prompt(p):
-    return raw_input('%s (and press enter to continue)' % p)
-
 ################################################################################
 
 class CdromModuleTest(unittest.TestCase):
-    def test_CD(self):
+    def todo_test_CD(self):
 
         # __doc__ (as of 2008-06-25) for pygame.cdrom.CD:
 
           # pygame.cdrom.CD(id): return CD
           # class to manage a cdrom drive
 
-        self.assert_(test_not_implemented())
+        self.fail()
 
-    def test_get_count(self):
+    def todo_test_get_count(self):
+
 
         # __doc__ (as of 2008-06-25) for pygame.cdrom.get_count:
 
           # pygame.cdrom.get_count(): return count
           # number of cd drives on the system
 
-        self.assert_(test_not_implemented())
+        self.fail()
 
-    def test_get_init(self):
+    def todo_test_get_init(self):
+
 
         # __doc__ (as of 2008-06-25) for pygame.cdrom.get_init:
 
           # pygame.cdrom.get_init(): return bool
           # true if the cdrom module is initialized
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_init(self):
+    def todo_test_init(self):
+
 
         # __doc__ (as of 2008-06-25) for pygame.cdrom.init:
 
           # pygame.cdrom.init(): return None
           # initialize the cdrom module
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_quit(self):
+    def todo_test_quit(self):
+
 
         # __doc__ (as of 2008-06-25) for pygame.cdrom.quit:
 
           # pygame.cdrom.quit(): return None
           # uninitialize the cdrom module
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
 class CDTypeTest(unittest.TestCase):
-    '|Tags:interactive|'
-
     def setUp(self):
         pygame.cdrom.init()
 
 
         # self.assert_(test_not_implemented())
 
-    def test_get_all(self):
+    def todo_test_get_all(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_all:
 
         
         # self.cd.init()
         
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_get_busy(self):
+    def todo_test_get_busy(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_busy:
 
           # CD.get_busy(): return bool
           # true if the drive is playing audio
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_get_current(self):
+    def todo_test_get_current(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_current:
 
           # CD.get_current(): return track, seconds
           # the current audio playback position
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_get_empty(self):
+    def todo_test_get_empty(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_empty:
 
           # CD.get_empty(): return bool
           # False if a cdrom is in the drive
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_get_id(self):
+    def todo_test_get_id(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_id:
 
           # CD.get_init(): return bool
           # true if this cd device initialized
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_get_init(self):
+    def todo_test_get_init(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_init:
 
           # CD.get_init(): return bool
           # true if this cd device initialized
 
-        self.assert_(test_not_implemented())
+        self.fail()
 
     def test_2_get_name(self):
 
                 question('Is %s the correct name for the cd drive?' % cd_name)
             )
 
-    def test_get_numtracks(self):
+    def todo_test_get_numtracks(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_numtracks:
 
           # CD.get_numtracks(): return count
           # the number of tracks on the cdrom
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_get_paused(self):
+    def todo_test_get_paused(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_paused:
 
           # CD.get_paused(): return bool
           # true if the drive is paused
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_get_track_audio(self):
+    def todo_test_get_track_audio(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_track_audio:
 
           # CD.get_track_audio(track): return bool
           # true if the cdrom track has audio data
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_get_track_length(self):
+    def todo_test_get_track_length(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_track_length:
 
           # CD.get_track_length(track): return seconds
           # length of a cdrom track
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_get_track_start(self):
+    def todo_test_get_track_start(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.get_track_start:
 
           # CD.get_track_start(track): return seconds
           # start time of a cdrom track
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_init(self):
+    def todo_test_init(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.init:
 
           # CD.init(): return None
           # initialize a cdrom drive for use
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_pause(self):
+    def todo_test_pause(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.pause:
 
           # CD.pause(): return None
           # temporarily stop audio playback
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_play(self):
+    def todo_test_play(self):
+
 
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.play:
 
           # CD.init(): return None
           # initialize a cdrom drive for use
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_quit(self):
+    def todo_test_quit(self):
+
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.quit:
 
           # CD.quit(): return None
           # uninitialize a cdrom drive for use
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_resume(self):
+    def todo_test_resume(self):
+
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.resume:
 
           # CD.resume(): return None
           # unpause audio playback
 
-        self.assert_(test_not_implemented()) 
+        self.fail()
 
-    def test_stop(self):
+    def todo_test_stop(self):
+
         # __doc__ (as of 2008-07-02) for pygame.cdrom.CD.stop:
 
           # CD.stop(): return None
           # stop audio playback
 
-        self.assert_(test_not_implemented())
+        self.fail()
 
 ################################################################################
 

File test/test_utils.py

 
 import tempfile, sys, pygame, time, os
 
-# from test import pystone
-
 ################################################################################
 
 trunk_dir = os.path.split(os.path.dirname(os.path.abspath(__file__)))[0]
 import test.unittest as unittest
 
 ############################### INCOMPLETE TESTS ###############################
+# TODO: PHASE THIS OUT
+# Just prefix TODO test names with todo_. 
+# eg def todo_test_sanity__is_overrated(self): self.fail()
+# Change test loader to load test_ and todo_ TestCase callables as tests
 
 fail_incomplete_tests = 0
 
 def get_tmp_dir():
     return tempfile.mkdtemp()
 
-
 ################################################################################
 
-# TOLERANCE in Pystones
-# kPS = 1000
-# TOLERANCE = 0.5*kPS 
+def question(q):
+    return raw_input('%s ' % q.rstrip(' ')).lower().strip() == 'y'
 
-# class DurationError(AssertionError): pass
-
-# def local_pystone():
-#     return pystone.pystones(loops=pystone.LOOPS)
-
-# def timedtest(max_num_pystones, current_pystone=local_pystone()):
-#     """ decorator timedtest """
-#     if not isinstance(max_num_pystones, float):
-#         max_num_pystones = float(max_num_pystones)
-
-#     def _timedtest(function):
-#         def wrapper(*args, **kw):
-#             start_time = time.time()
-#             try:
-#                 return function(*args, **kw)
-#             finally:
-#                 total_time = time.time() - start_time
-#                 if total_time == 0:
-#                     pystone_total_time = 0
-#                 else:
-#                     pystone_rate = current_pystone[0] / current_pystone[1]
-#                     pystone_total_time = total_time / pystone_rate
-#                 if pystone_total_time > (max_num_pystones + TOLERANCE):
-#                     raise DurationError((('Test too long (%.2f Ps, '
-#                                         'need at most %.2f Ps)')
-#                                         % (pystone_total_time,
-#                                             max_num_pystones)))
-#         return wrapper
-
-#     return _timedtest
+def prompt(p):
+    return raw_input('%s (and press enter to continue) ' % p.rstrip(' '))
 
 #################################### HELPERS ###################################
 
     """
     return (
          (rect.left is not 0 and [(rect.left-1, rect.top)] or []) +
-        [ rect.topright,                                          
+        [ rect.topright,
           rect.bottomleft,                                             
           rect.bottomright]  
     ) 

File test/util/gen_stubs.py

 from __future__ import with_statement
 from optparse import OptionParser
 from inspect import isclass, ismodule, getdoc, isgetsetdescriptor, getmembers
-from unittest import TestCase
 
 import pygame, sys, datetime, re, types
 import relative_indentation
 
 from os.path import normpath, join, dirname, abspath
 
-sys.path.append( abspath(normpath( join(dirname(__file__), '../') )) )
+for relpath in ('../../','../'):
+    sys.path.insert(0, abspath(normpath( join(dirname(__file__), relpath) )) )
+
+from test.unittest import TestCase
 
 #################################### IGNORES ###################################
 
     pygame.sprite.GroupSingle,
     pygame.sprite.RenderUpdates,
     pygame.sprite.Group,
+    
+    pygame.image.tostring,
 ])
 
 # pygame.sprite.Sprite.__module__ = 'pygame.sprite' 
 }
 
 MUST_INSTANTIATE = {
-    
     # BaseType / Helper               # (Instantiator / Args) / Callable
 
-    pygame.cdrom.CDType            :  (pygame.cdrom.CD, (0,)),
+    pygame.cdrom.CDType            :  (pygame.cdrom.CD,      (0,)),
     pygame.mixer.ChannelType       :  (pygame.mixer.Channel, (0,)),
-    pygame.time.Clock              :  (pygame.time.Clock, ()),
+    pygame.time.Clock              :  (pygame.time.Clock,    ()),
 
     # pygame.event.Event         :  None,
     # pygame.joystick.Joystick   :  None,
     except TypeError: obj_name = obj_name.__name__
     return not obj_name.startswith(('__','_'))
 
-def is_test(f):
-    return f.__name__.startswith('test_')
-
 def get_callables(obj, if_of = None, check_where_defined=False):
     publics = (getattr(obj, x) for x in dir(obj) if is_public(x))
     callables = (x for x in publics if callable(x) or isgetsetdescriptor(x))
     if check_where_defined:
         callables = (c for c in callables if ( 'pygame' in c.__module__ or
                     ('__builtin__' == c.__module__ and isclass(c)) )
-                    and REAL_HOMES.get(c, 0) in (0, obj))
+                    and REAL_HOMES.get(c, 0) in (obj, 0))
 
     if if_of:
         callables = (x for x in callables if if_of(x)) # isclass, ismethod etc
 def get_class_from_test_case(TC):
     TC = TC.__name__
     if 'Type' in TC:
-        return '.' + TC[:TC.index('Type')]
+        return '.' + TC[:TC.rindex('Type')]
 
 def names_of(*args):
     return tuple(map(lambda o: getattr(o, "__name__", str(o)), args))
 ################################################################################
 
 def test_stub(f, module, parent_class = None):
-    test_name = 'test_%s' % f.__name__
+    test_name = 'todo_test_%s' % f.__name__
     unit_name = callable_name(module, parent_class, f)
 
     stub = STUB_TEMPLATE.render (
 
 ################################################################################
 
+TEST_NAME_RE = re.compile(r"test[_\d]+(.*)")
+
+def is_test(f):
+    return f.__name__.startswith(('test_', 'todo_'))
+
+def get_tested_from_testname(test):
+    tn = test.__name__
+    separated = tn.rfind('__')
+    if separated != -1: tn = tn[:separated]
+    return TEST_NAME_RE.search(tn).group(1)
+
+################################################################################
+
 def already_tested_in_module(module):
     already = []
 
         class_tested = get_class_from_test_case(class_) or ''
 
         for test in get_callables(class_, is_test):
-            fname = test.__name__[5:].split('__')[0]
+            fname = get_tested_from_testname(test)
             already.append("%s%s.%s" % (mod_name, class_tested, fname))
 
     return already

File test_runner.py

 ################################################################################
 
-#TODO: clean up imports
-
 import test.unittest as unittest
 
 import sys, os, re, StringIO, time, optparse
               "Run test T times, giving average time")
 
 opt_parser.add_option (
-     "-e",  "--exclude",
+     "-e",  "--exclude", default = '',
      help   = "exclude tests containing any of TAGS" )
 
 opt_parser.add_option (
             # would this effect the original dict? TODO
             results['raw_return'] = ''.join(raw_return.splitlines(1)[:5])
             failures.append( COMPLETE_FAILURE_TEMPLATE % results )
-            all_dot += 'E'
+            all_dots += 'E'
             continue
 
         dots = DOTS.search(output).group(1)
 
     test = unittest.defaultTestLoader.loadTestsFromName(module)
     suite.addTest(test)
-        
+
     output = StringIO.StringIO()
     runner = unittest.TextTestRunner(stream = output)
     

File unittest_patch.py

     
     ########################################################################
     # Pre run:
-        
+
+        #TODO: only redirect output if not tagged interactive
+
         result.tests[self.dot_syntax_name()] = {}
         tests = result.tests[self.dot_syntax_name()]
         (realerr, realout), (stderr, stdout) =  redirect_output()
-        # restore_output(realerr, realout)      # DEBUG
+
+        if 0 or 'interactive' in get_tags(testMethod):       # DEBUG
+            restore_output(realerr, realout)
 
         t = time.time()
 
         t = (time.time() -t) / self.times_run
         
         restore_output(realerr, realout)
-        
+
         tests["time"]   = t
         tests["stdout"] = StringIOContents(stdout)
         tests["stderr"] = StringIOContents(stderr)
     self.testsRun   = 0
     self.shouldStop = 0
 
-
-# TODO: all this is available in the traceback object
+# TODO: all this is available in the traceback object err
 FILE_LINENUMBER_RE = re.compile(r'File "([^"]+)", line ([0-9]+)')
 
 def errorHandling(key):
 # Exclude by tags
 #
 
-TAGS_RE = re.compile(r"\|[tT]ags:([ a-zA-Z,0-9_\n]+)\|", re.M)
+TAGS_RE = re.compile(r"\|[tT]ags:(-?[ a-zA-Z,0-9_\n]+)\|", re.M)
 
-def get_tags(obj):
-    tags = TAGS_RE.search(getdoc(obj) or '')
-    return tags and [t.strip() for t in tags.group(1).split(',')] or []
+class TestTags:
+    def __init__(self):
+        self.memoized = {}
+        self.parent_modules = {}
+
+    def get_parent_module(self, class_):
+        while class_ not in self.parent_modules:
+            self.parent_modules[class_] = __import__(class_.__module__)
+        return self.parent_modules[class_]
+
+    def __call__(self, obj):
+        while obj not in self.memoized:
+            parent_class  = obj.im_class
+            parent_module = self.get_parent_module(parent_class)
+
+            module_tags = getattr(parent_module, '__tags__', [])
+            class_tags  = getattr(parent_class,  '__tags__', [])
+
+            tags = TAGS_RE.search(getdoc(obj) or '')
+            if tags: test_tags = [t.strip() for t in tags.group(1).split(',')]
+            else:    test_tags = []
+        
+            combined = set()
+            for tags in (module_tags, class_tags, test_tags):
+                if not tags: continue
+        
+                add    = set(t for t in tags if not t.startswith('-'))
+                remove = set(t[1:] for t in tags if t not in add)
+        
+                if add:     combined.update(add)
+                if remove:  combined.difference_update(remove)
+    
+            self.memoized[obj] = combined
+
+        return self.memoized[obj]
+
+get_tags = TestTags()
+
+################################################################################
 
 def getTestCaseNames(self, testCaseClass):
-    def test_wanted(attrname, testCaseClass=testCaseClass, 
-                                    prefix=self.testMethodPrefix):
-                                    #TODO: ('test_','todo_')
+    def test_wanted(attrname, testCaseClass=testCaseClass,
+                              prefix=self.testMethodPrefix):
+        if not attrname.startswith(prefix): return False
+        else:
+            actual_attr = getattr(testCaseClass, attrname)
+            return (
+                 callable(actual_attr) and
+                 not [t for t in  get_tags(actual_attr) if t in self.exclude]
+            )
 
-        actual_attr = getattr(testCaseClass, attrname)
-        filtered = bool([t for t in get_tags(actual_attr) if t in self.exclude])
-        return ( attrname.startswith(prefix) and callable(actual_attr)
-                 and not filtered )
+    # TODO:
 
+    # Replace test_not_implemented mechanism with technique that names the tests
+    # todo_test_xxxxxx, then when wanting to fail them, loads any members that
+    # startswith(test_prefix)
+    
+    # REGEX FOR TEST_NOT_IMPLEMENTED
+    # SEARCH:
+    #    def (test_[^ ]+)((?:\s+#.*\n?)+\s+)self\.assert_\(test_not_implemented\(\)\)
+    # REPLACE:
+    #    def todo_\1\2self.fail()
+    
     testFnNames = filter(test_wanted, dir(testCaseClass))
-    
+
     for baseclass in testCaseClass.__bases__:
         for testFnName in self.getTestCaseNames(baseclass):
             if testFnName not in testFnNames:  # handle overridden methods
 
 def patch(options):
     if options.incomplete:
-        unittest.TestLoader.testMethodPrefix = tuple (
-            list(self.testMethodPrefix) + ['todo_']
+        unittest.TestLoader.testMethodPrefix = (
+            unittest.TestLoader.testMethodPrefix, 'todo_'
         )
-    
-    # Tag exclusion
-    if options.exclude:
-        unittest.TestLoader.getTestCaseNames = getTestCaseNames
-        unittest.TestLoader.exclude = (
-            [e.strip() for e in options.exclude.split(',')] )
+
+    unittest.TestLoader.getTestCaseNames = getTestCaseNames
+    unittest.TestLoader.exclude = (
+        [e.strip() for e in options.exclude.split(',')] )
 
     # Timing
     unittest.TestCase.times_run = options.timings