Commits

Anonymous committed 1a6bbfc

added test order randomization to run_tests.py

  • Participants
  • Parent commits 2f4e26c

Comments (0)

Files changed (3)

File run_tests.py

 
 import test.unittest as unittest
 
-import sys, os, re, subprocess, time, optparse
+import sys
+import os
+import re
+import subprocess
+import time
+import optparse
 import pygame.threads, pygame
+import random
 
 from test_runner import prepare_test_env, run_test, combine_results, \
                         test_failures, get_test_results, from_namespace, \
 main_dir, test_subdir, fake_test_subdir = prepare_test_env()
 test_runner_py = os.path.join(main_dir, "test_runner.py")
 
-import test_utils, unittest_patch
+import test_utils
+import unittest_patch
 
 ################################### CONSTANTS ##################################
 # Defaults:
 else:
     working_dir = main_dir
 
-test_env = {"PYTHONPATH": test_subdir}     #TODO:  append to PYTHONPATH
-try:
-    # Required by Python 2.6 on Windows.
-    test_env["SystemRoot"] = os.environ["SystemRoot"]
-except KeyError:
+test_env = {"PYTHONPATH": test_subdir}     #TODO:  append to PYTHONPATH
+try:
+    # Required by Python 2.6 on Windows.
+    test_env["SystemRoot"] = os.environ["SystemRoot"]
+except KeyError:
     pass
 os.chdir(working_dir)
 
                 test_modules.append(match)
 
 ################################################################################
+# Meta results
+
+meta_results = {}
+
+################################################################################
+# Randomization
+
+if options.randomize or options.seed:
+    t = time.time()
+    meta_results['random_seed'] = t
+    if options.seed: random.seed(options.seed)
+    else:            random.seed(t)
+        
+################################################################################
 # Single process mode
 
 if not options.subprocess:
             )
     else: tmap = map
 
-    results = {}
     t = time.time()
 
     for module, cmd, (return_code, raw_return) in tmap(sub_test, test_modules):

File test_runner.py

 
 import unittest_patch
 from unittest_patch import StringIOContents
+# from safe_eval import safe_eval as eval
 
 ################################################################################
 
      help   = "dump all results not just errors eg. -da" )
 
 opt_parser.add_option (
-     "-H",  "--human", action = 'store_true',
-     help   = "dump results as dict ready to eval if unsure "
-              "that pieced together results are correct "
-              "(subprocess mode)" )
+     "-r",  "--randomize", action = 'store_true',
+     help   = "randomize order of tests" )
+
+opt_parser.add_option (
+     "-S",  "--seed", type = 'int',
+     help   = "seed randomizer" )
 
 opt_parser.add_option (
      "-m",  "--multi_thread", metavar = 'THREADS', type = 'int',
     failures = []
 
     for module, results in sorted(all_results.items()):
+        if module == '__meta__': continue
         output, return_code, raw_return = map (
             results.get, ('output','return_code', 'raw_return')
         )
 
     output = StringIO.StringIO()
     runner = unittest.TextTestRunner(stream = output)
-    
+
     results = runner.run(suite)
     output  = StringIOContents(output)
 
     tests     = results.tests
 
     results   = {module:from_namespace(locals(), RESULTS_TEMPLATE)}
-    
+
     if options.subprocess:
         print TEST_RESULTS_START
         print pformat(results)

File unittest_patch.py

 ################################################################################
 
 import test.unittest as unittest
-import re, time, sys, StringIO
+import re
+import time
+import sys 
+import StringIO
+import random
+
 from inspect import getdoc
 
 # This is needed for correct tracebacks
     if result is None: result = self.defaultTestResult()
     result.startTest(self)
     testMethod = getattr(self, self._testMethodName)
+    
+    print self._testMethodName
     try:
 
     ########################################################################
     # Pre run:
-
         #TODO: only redirect output if not tagged interactive
 
-        result.tests[self.dot_syntax_name()] = {}
+        result.tests[self.dot_syntax_name()] = {
+            'times' : [],
+        }
+        
         tests = result.tests[self.dot_syntax_name()]
         (realerr, realout), (stderr, stdout) =  redirect_output()
-
         test_tags = list(get_tags(testMethod))
 
         if 0 or 'interactive' in test_tags:       # DEBUG
             restore_output(realerr, realout)
 
-        t = time.time()
-
     ########################################################################
 
         for i in range(self.times_run):
+            t = time.time()
+            
             try:
                 self.setUp()
             except KeyboardInterrupt:
             except:
                 result.addError(self, self._exc_info())
                 ok = False
-    
+            
+            
+            tests["times"] += [time.time() -t]
+            
             if ok:
                 if i == 0:
                     result.addSuccess(self)
             else: break
-
+            
+            
+            
     ########################################################################
     # Post run
 
-        t = (time.time() -t) / self.times_run
-        
         restore_output(realerr, realout)
 
-        tests["time"]   = t
         tests["stdout"] = StringIOContents(stdout)
         tests["stderr"] = StringIOContents(stderr)
         tests["tags"]   = test_tags
         self.parent_modules = {}
 
     def get_parent_module(self, class_):
-        while class_ not in self.parent_modules:
+        if class_ not in self.parent_modules:
             self.parent_modules[class_] = __import__(class_.__module__)
         return self.parent_modules[class_]
 
     def __call__(self, obj):
-        while obj not in self.memoized:
+        if obj not in self.memoized:
             parent_class  = obj.im_class
             parent_module = self.get_parent_module(parent_class)
 
 get_tags = TestTags()
 
 ################################################################################
-
+# unittest.TestLoader
+#
 def getTestCaseNames(self, testCaseClass):
     def test_wanted(attrname, testCaseClass=testCaseClass,
                               prefix=self.testMethodPrefix):
             if testFnName not in testFnNames:  # handle overridden methods
                 testFnNames.append(testFnName)
 
-    if self.sortTestMethodsUsing:
+    if self.randomize_tests:
+        random.shuffle(testFnNames)
+    elif self.sortTestMethodsUsing:
         testFnNames.sort(self.sortTestMethodsUsing)
 
     return testFnNames
         unittest.TestLoader.testMethodPrefix = (
             unittest.TestLoader.testMethodPrefix, 'todo_'
         )
+    
+    unittest.TestLoader.randomize_tests = options.randomize or options.seed
 
     unittest.TestLoader.getTestCaseNames = getTestCaseNames
     unittest.TestLoader.exclude = (