Commits

Rob Managan committed 354524a Merge

update to latest SCons commits

Comments (0)

Files changed (17)

QMTest/TestCmd.py

 
 
 
+import subprocess
+
 try:
-    import subprocess
-except ImportError:
-    # The subprocess module doesn't exist in this version of Python,
-    # so we're going to cobble up something that looks just enough
-    # like its API for our purposes below.
-    import popen2
-    subprocess = types.ModuleType('subprocess')
-
-    subprocess.PIPE = 'PIPE'
-    subprocess.STDOUT = 'STDOUT'
-    subprocess.mswindows = (sys.platform == 'win32')
-
-    class Popen(popen2.Popen3, popen2.Popen4):
-        universal_newlines = 1
-        def __init__(self, command, **kw):
-            if kw.get('stderr') == 'STDOUT':
-                popen2.Popen4.__init__(self, command, 1)
-            else:
-                popen2.Popen3.__init__(self, command, 1)
-            self.stdin = self.tochild
-            self.stdout = self.fromchild
-            self.stderr = self.childerr
-        def communicate(self, input=None):
-            if input:
-                self.stdin.write(input)
-            self.stdin.close()
-            out = self.stdout.read()
-            if self.stderr is None:
-                err = None
-            else:
-                err = self.stderr.read()
-            self.stdout.close()
-            if self.stderr is not None:
-                self.stderr.close()
-            self.returncode = self.wait()
-            return (out, err)
+    subprocess.Popen.terminate
+except AttributeError:
+    if sys.platform == 'win32':
+        import win32process
+        def terminate(self):
+            win32process.TerminateProcess(self._handle, 1)
+    else:
         def terminate(self):
             os.kill(self.pid, signal.SIGTERM)
-        def wait(self, *args, **kw):
-            resultcode = popen2.Popen3.wait(self, *args, **kw)
-            if os.WIFSIGNALED(resultcode):
-                return (- os.WTERMSIG(resultcode))
-            elif os.WIFEXITED(resultcode):
-                return os.WEXITSTATUS(resultcode)
-            else:
-                return None
-
-    subprocess.Popen = Popen
-else:
-    try:
-        subprocess.Popen.terminate
-    except AttributeError:
-        if sys.platform == 'win32':
-            import win32process
-            def terminate(self):
-                win32process.TerminateProcess(self._handle, 1)
-        else:
-            def terminate(self):
-                os.kill(self.pid, signal.SIGTERM)
-        method = types.MethodType(terminate, None, subprocess.Popen)
-        setattr(subprocess.Popen, 'terminate', method)
+    method = types.MethodType(terminate, None, subprocess.Popen)
+    setattr(subprocess.Popen, 'terminate', method)
 
 
 

QMTest/TestCmdTests.py

         except KeyError:
             pass
 
+import subprocess
+
 try:
-    import subprocess
-except ImportError:
-    # The subprocess module doesn't exist in this version of Python,
-    # so we're going to cobble up something that looks just enough
-    # like its API for our purposes below.
-    import popen2
-    subprocess = types.ModuleType('subprocess')
-
-    subprocess.PIPE = 'PIPE'
-    subprocess.STDOUT = 'STDOUT'
-    subprocess.mswindows = (sys.platform == 'win32')
-
-    class Popen(popen2.Popen3, popen2.Popen4):
-        universal_newlines = 1
-        def __init__(self, command, **kw):
-            if kw.get('stderr') == 'STDOUT':
-                popen2.Popen4.__init__(self, command, 1)
-            else:
-                popen2.Popen3.__init__(self, command, 1)
-            self.stdin = self.tochild
-            self.stdout = self.fromchild
-            self.stderr = self.childerr
-        def communicate(self, input=None):
-            if input:
-                self.stdin.write(input)
-            self.stdin.close()
-            out = self.stdout.read()
-            if self.stderr is None:
-                err = None
-            else:
-                err = self.stderr.read()
-            self.stdout.close()
-            if self.stderr is not None:
-                self.stderr.close()
-            self.returncode = self.wait()
-            return (out, err)
+    subprocess.Popen.terminate
+except AttributeError:
+    if sys.platform == 'win32':
+        import win32process
+        def terminate(self):
+            win32process.TerminateProcess(self._handle, 1)
+    else:
         def terminate(self):
             os.kill(self.pid, signal.SIGTERM)
-        def wait(self, *args, **kw):
-            resultcode = popen2.Popen3.wait(self, *args, **kw)
-            if os.WIFEXITED(resultcode):
-                return os.WEXITSTATUS(resultcode)
-            elif os.WIFSIGNALED(resultcode):
-                return os.WTERMSIG(resultcode)
-            else:
-                return None
-
-    subprocess.Popen = Popen
-else:
-    try:
-        subprocess.Popen.terminate
-    except AttributeError:
-        if sys.platform == 'win32':
-            import win32process
-            def terminate(self):
-                win32process.TerminateProcess(self._handle, 1)
-        else:
-            def terminate(self):
-                os.kill(self.pid, signal.SIGTERM)
-        method = types.MethodType(terminate, None, subprocess.Popen)
-        setattr(subprocess.Popen, 'terminate', method)
+    method = types.MethodType(terminate, None, subprocess.Popen)
+    setattr(subprocess.Popen, 'terminate', method)
 
 class ExitError(Exception):
     pass

QMTest/TestCommon.py

             print "Unwritable files: `%s'" % "', `".join(unwritable)
         self.fail_test(missing + unwritable)
 
-    def must_contain(self, file, required, mode = 'rb'):
+    def must_contain(self, file, required, mode = 'rb', find = None):
         """Ensures that the specified file contains the required text.
         """
         file_contents = self.read(file, mode)
-        contains = (file_contents.find(required) != -1)
+        if find is None:
+            def find(o, l):
+                try:
+                    return o.index(l)
+                except ValueError:
+                    return None
+        contains = find(file_contents, required)
         if not contains:
             print "File `%s' does not contain required string." % file
             print self.banner('Required string ')
                 except ValueError:
                     return None
         missing = []
+        if is_List(output):
+            output = '\n'.join(output)
+
         for line in lines:
             if find(output, line) is None:
                 missing.append(line)
         sys.stdout.flush()
         self.fail_test()
 
-    def must_contain_lines(self, lines, output, title=None):
+    def must_contain_lines(self, lines, output, title=None, find = None):
         # Deprecated; retain for backwards compatibility.
-        return self.must_contain_all_lines(output, lines, title)
+        return self.must_contain_all_lines(output, lines, title, find)
 
     def must_exist(self, *files):
         """Ensures that the specified file(s) must exist.  An individual
             self.diff(expect, file_contents, 'contents ')
             raise
 
-    def must_not_contain(self, file, banned, mode = 'rb'):
+    def must_not_contain(self, file, banned, mode = 'rb', find = None):
         """Ensures that the specified file doesn't contain the banned text.
         """
         file_contents = self.read(file, mode)
-        contains = (file_contents.find(banned) != -1)
+        if find is None:
+            def find(o, l):
+                try:
+                    return o.index(l)
+                except ValueError:
+                    return None
+        contains = find(file_contents, banned)
         if contains:
             print "File `%s' contains banned string." % file
             print self.banner('Banned string ')
             sys.stdout.write(output)
             self.fail_test()
 
-    def must_not_contain_lines(self, lines, output, title=None):
-        return self.must_not_contain_any_line(output, lines, title)
+    def must_not_contain_lines(self, lines, output, title=None, find=None):
+        return self.must_not_contain_any_line(output, lines, title, find)
 
     def must_not_exist(self, *files):
         """Ensures that the specified file(s) must not exist.

QMTest/TestSCons.py

         str = str.replace(c, '\\' + c)
     return str
 
+#
+# Helper functions that we use as a replacement to the default re.match
+# when searching for special strings in stdout/stderr.
+#
+def search_re(out, l):
+    """ Search the regular expression 'l' in the output 'out'
+        and return the start index when successful.
+    """
+    m = re.search(l, out)
+    if m:
+        return m.start()
+    
+    return None
 
+def search_re_in_list(out, l):
+    """ Search the regular expression 'l' in each line of
+        the given string list 'out' and return the line's index
+        when successful.
+    """
+    for idx, o in enumerate(out):
+        m = re.search(l, o)
+        if m:
+            return idx
+    
+    return None
+
+#
+# Helpers for handling Python version numbers
+#
 def python_version_string():
     return sys.version.split()[0]
 

QMTest/TestSCons_time.py

 
 from TestCommon import *
 from TestCommon import __all__
+# some of the scons_time tests may need regex-based matching:
+from TestSCons import search_re, search_re_in_list
 
 __all__.extend([ 'TestSCons_time',
                ])
 #
 #       -n              No execute, just print command lines.
 #
-#       -o file         Print test results to the specified file.
-#                       The --xml option specifies the
-#                       output format.
+#       -o file         Save screen output to the specified log file.
 #
 #       -P Python       Use the specified Python interpreter.
 #
 #
 #       -x scons        The scons script to use for tests.
 #
-#       --xml           Print test results to an output file (specified
-#                       by the -o option) in an SCons-specific XML format.
+#       --xml file      Save test results to the specified file in an
+#                       SCons-specific XML format.
 #                       This is (will be) used for reporting results back
 #                       to a central SCons test monitoring infrastructure.
 #
 import sys
 import time
 
+try:
+    import threading
+    import Queue                # 2to3: rename to queue
+    threading_ok = True
+except ImportError:
+    print "Can't import threading or queue"
+    threading_ok = False
+
 cwd = os.getcwd()
 
-all = 0
 baseline = 0
 builddir = os.path.join(cwd, 'build')
 external = 0
 debug = ''
 execute_tests = 1
-format = None
+jobs = 1
 list_only = None
 printcommand = 1
 package = None
 python3incompatibilities = None
 scons = None
 scons_exec = None
-outputfile = None
+qmtest = None
 testlistfile = None
 version = ''
 print_times = None
   -k, --no-progress           Suppress count and percent progress messages.
   -l, --list                  List available tests and exit.
   -n, --no-exec               No execute, just print command lines.
-  --noqmtest                  Execute tests directly, not using QMTest.
   --nopipefiles               Doesn't use the "file pipe" workaround for subprocess.Popen()
                               for starting tests. WARNING: Only use this when too much file
                               traffic is giving you trouble AND you can be sure that none of
                               your tests create output that exceed 65K chars! You might
                               run into some deadlocks else.
-  -o FILE, --output FILE      Print test results to FILE.
+  -o FILE, --output FILE      Save the output from a test run to the log file.
   -P Python                   Use the specified Python interpreter.
   -p PACKAGE, --package PACKAGE
                               Test against the specified PACKAGE:
                                 tar-gz        .tar.gz distribution
                                 zip           .zip distribution
   --passed                    Summarize which tests passed.
-  --qmtest                    Run using the QMTest harness.
+  --qmtest                    Run using the QMTest harness (deprecated).
   -q, --quiet                 Don't print the test being executed.
   -s, --short-progress        Short progress, prints only the command line
                               and a percentage value, based on the total and
                                 3 = print commands and all output.
   -X                          Test script is executable, don't feed to Python.
   -x SCRIPT, --exec SCRIPT    Test SCRIPT.
-  --xml                       Print results in SCons XML format.
+  --xml file                  Save results to file in SCons XML format.
 
 Environment Variables:
 
   TESTCMD_VERBOSE: turn on verbosity in TestCommand
 """
 
-opts, args = getopt.getopt(sys.argv[1:], "3ab:def:hklno:P:p:qsv:Xx:t",
-                            ['all', 'baseline=', 'builddir=',
+
+# "Pass-through" option parsing -- an OptionParser that ignores
+# unknown options and lets them pile up in the leftover argument
+# list.  Useful to gradually port getopt to optparse.
+
+from optparse import OptionParser, BadOptionError
+
+class PassThroughOptionParser(OptionParser):
+    def _process_long_opt(self, rargs, values):
+        try:
+            OptionParser._process_long_opt(self, rargs, values)
+        except BadOptionError, err:
+            self.largs.append(err.opt_str)
+    def _process_short_opts(self, rargs, values):
+        try:
+            OptionParser._process_short_opts(self, rargs, values)
+        except BadOptionError, err:
+            self.largs.append(err.opt_str)
+
+parser = PassThroughOptionParser(add_help_option=False)
+parser.add_option('-a', '--all', action='store_true',
+                      help="Run all tests.")
+parser.add_option('-o', '--output',
+                      help="Save the output from a test run to the log file.")
+parser.add_option('--xml',
+                      help="Save results to file in SCons XML format.")
+(options, args) = parser.parse_args()
+
+#print "options:", options
+#print "args:", args
+
+
+opts, args = getopt.getopt(args, "3b:def:hj:klnP:p:qsv:Xx:t",
+                            ['baseline=', 'builddir=',
                              'debug', 'external', 'file=', 'help', 'no-progress',
-                             'list', 'no-exec', 'noqmtest', 'nopipefiles', 'output=',
+                             'jobs=',
+                             'list', 'no-exec', 'nopipefiles',
                              'package=', 'passed', 'python=', 'qmtest',
                              'quiet', 'short-progress', 'time',
                              'version=', 'exec=',
-                             'verbose=', 'xml'])
+                             'verbose='])
 
 for o, a in opts:
     if o in ['-3']:
         python3incompatibilities = 1
-    elif o in ['-a', '--all']:
-        all = 1
     elif o in ['-b', '--baseline']:
         baseline = a
     elif o in ['--builddir']:
     elif o in ['-h', '--help']:
         print helpstr
         sys.exit(0)
+    elif o in ['-j', '--jobs']:
+        jobs = int(a)
     elif o in ['-k', '--no-progress']:
         print_progress = 0
     elif o in ['-l', '--list']:
         list_only = 1
     elif o in ['-n', '--no-exec']:
         execute_tests = None
-    elif o in ['--noqmtest']:
-        qmtest = None
     elif o in ['--nopipefiles']:
         allow_pipe_files = False
-    elif o in ['-o', '--output']:
-        if a != '-' and not os.path.isabs(a):
-            a = os.path.join(cwd, a)
-        outputfile = a
     elif o in ['-p', '--package']:
         package = a
     elif o in ['--passed']:
         scons_exec = 1
     elif o in ['-x', '--exec']:
         scons = a
-    elif o in ['--xml']:
-        format = o
 
-if not args and not all and not testlistfile:
+if not args and not options.all and not testlistfile:
     sys.stderr.write("""\
 runtest.py:  No tests were specified.
              List one or more tests on the command line, use the
 """)
     sys.exit(1)
 
+
+# --- setup stdout/stderr ---
+class Unbuffered(object):
+    def __init__(self, file):
+        self.file = file
+        self.softspace = 0  ## backward compatibility; not supported in Py3k
+    def write(self, arg):
+        self.file.write(arg)
+        self.file.flush()
+    def __getattr__(self, attr):
+        return getattr(self.file, attr)
+
+sys.stdout = Unbuffered(sys.stdout)
+sys.stderr = Unbuffered(sys.stderr)
+
+if options.output:
+    logfile = open(options.output, 'w')
+    class Tee(object):
+        def __init__(self, openfile, stream):
+            self.file = openfile
+            self.stream = stream
+        def write(self, data):
+            self.file.write(data)
+            self.stream.write(data)
+    sys.stdout = Tee(logfile, sys.stdout)
+    sys.stderr = Tee(logfile, sys.stderr)
+
+# --- define helpers ----
 if sys.platform in ('win32', 'cygwin'):
 
     def whereis(file):
                     return f
         return None
 
-# See if --qmtest or --noqmtest specified
-try:
-    qmtest
-except NameError:
-    qmtest = None
-
 sp.append(builddir)
 sp.append(cwd)
 
     s = s.replace('\\', '\\\\')
     return s
 
-# Try to use subprocess instead of the more low-level
-# spawn command...
-use_subprocess = True
-try:
-    import subprocess
-except:
-    use_subprocess = False
-    
-if use_subprocess:
-    if not suppress_stdout and not suppress_stderr:
-        # Without any output suppressed, we let the subprocess
-        # write its stuff freely to stdout/stderr.
+
+import subprocess
+
+if not suppress_stdout and not suppress_stderr:
+    # Without any output suppressed, we let the subprocess
+    # write its stuff freely to stdout/stderr.
+    def spawn_it(command_args):
+        p = subprocess.Popen(' '.join(command_args),
+                             shell=True)
+        return (None, None, p.wait())
+else:
+    # Else, we catch the output of both pipes...
+    if allow_pipe_files:
+        # The subprocess.Popen() suffers from a well-known
+        # problem. Data for stdout/stderr is read into a 
+        # memory buffer of fixed size, 65K which is not very much.
+        # When it fills up, it simply stops letting the child process
+        # write to it. The child will then sit and patiently wait to
+        # be able to write the rest of its output. Hang! 
+        # In order to work around this, we follow a suggestion
+        # by Anders Pearson in
+        #   http://http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-is-your-enemy/
+        # and pass temp file objects to Popen() instead of the ubiquitous
+        # subprocess.PIPE. 
+        def spawn_it(command_args):
+            # Create temporary files
+            import tempfile
+            tmp_stdout = tempfile.TemporaryFile(mode='w+t')
+            tmp_stderr = tempfile.TemporaryFile(mode='w+t')
+            # Start subprocess...
+            p = subprocess.Popen(' '.join(command_args),
+                                 stdout=tmp_stdout,
+                                 stderr=tmp_stderr,
+                                 shell=True)
+            # ... and wait for it to finish.
+            ret = p.wait()
+            
+            try:
+                # Rewind to start of files
+                tmp_stdout.seek(0)
+                tmp_stderr.seek(0)
+                # Read output
+                spawned_stdout = tmp_stdout.read()
+                spawned_stderr = tmp_stderr.read()
+            finally:
+                # Remove temp files by closing them
+                tmp_stdout.close()
+                tmp_stderr.close()
+                
+            # Return values
+            return (spawned_stderr, spawned_stdout, ret)
+        
+    else:
+        # We get here only if the user gave the '--nopipefiles'
+        # option, meaning the "temp file" approach for
+        # subprocess.communicate() above shouldn't be used.
+        # He hopefully knows what he's doing, but again we have a
+        # potential deadlock situation in the following code:
+        #   If the subprocess writes a lot of data to its stderr,
+        #   the pipe will fill up (nobody's reading it yet) and the
+        #   subprocess will wait for someone to read it. 
+        #   But the parent process is trying to read from stdin
+        #   (but the subprocess isn't writing anything there).  
+        #   Hence a deadlock.
+        # Be dragons here! Better don't use this!
         def spawn_it(command_args):
             p = subprocess.Popen(' '.join(command_args),
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE,
                                  shell=True)
-            return (None, None, p.wait())
-    else:
-        # Else, we catch the output of both pipes...
-        if allow_pipe_files:
-            # The subprocess.Popen() suffers from a well-known
-            # problem. Data for stdout/stderr is read into a 
-            # memory buffer of fixed size, 65K which is not very much.
-            # When it fills up, it simply stops letting the child process
-            # write to it. The child will then sit and patiently wait to
-            # be able to write the rest of its output. Hang! 
-            # In order to work around this, we follow a suggestion
-            # by Anders Pearson in
-            #   http://http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-is-your-enemy/
-            # and pass temp file objects to Popen() instead of the ubiquitous
-            # subprocess.PIPE. 
-            def spawn_it(command_args):
-                # Create temporary files
-                import tempfile
-                tmp_stdout = tempfile.TemporaryFile(mode='w+t')
-                tmp_stderr = tempfile.TemporaryFile(mode='w+t')
-                # Start subprocess...
-                p = subprocess.Popen(' '.join(command_args),
-                                     stdout=tmp_stdout,
-                                     stderr=tmp_stderr,
-                                     shell=True)
-                # ... and wait for it to finish.
-                ret = p.wait()
-                
-                try:
-                    # Rewind to start of files
-                    tmp_stdout.seek(0)
-                    tmp_stderr.seek(0)
-                    # Read output
-                    spawned_stdout = tmp_stdout.read()
-                    spawned_stderr = tmp_stderr.read()
-                finally:
-                    # Remove temp files by closing them
-                    tmp_stdout.close()
-                    tmp_stderr.close()
-                    
-                # Return values
-                return (spawned_stderr, spawned_stdout, ret)
-            
-        else:
-            # We get here only if the user gave the '--nopipefiles'
-            # option, meaning the "temp file" approach for
-            # subprocess.communicate() above shouldn't be used.
-            # He hopefully knows what he's doing, but again we have a
-            # potential deadlock situation in the following code:
-            #   If the subprocess writes a lot of data to its stderr,
-            #   the pipe will fill up (nobody's reading it yet) and the
-            #   subprocess will wait for someone to read it. 
-            #   But the parent process is trying to read from stdin
-            #   (but the subprocess isn't writing anything there).  
-            #   Hence a deadlock.
-            # Be dragons here! Better don't use this!
-            def spawn_it(command_args):
-                p = subprocess.Popen(' '.join(command_args),
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE,
-                                     shell=True)
-                spawned_stdout = p.stdout.read()
-                spawned_stderr = p.stderr.read()
-                return (spawned_stderr, spawned_stdout, p.wait())
-else:
-    has_subprocess = False
-    # Set up lowest-common-denominator spawning of a process on both Windows
-    # and non-Windows systems that works all the way back to Python 1.6
-    def spawn_it(command_args):
-        command = command_args[0]
-        command_args = [escape(c) for c in command_args]
-        return (None, None, os.spawnv(os.P_WAIT, command, command_args))
+            spawned_stdout = p.stdout.read()
+            spawned_stderr = p.stderr.read()
+            return (spawned_stderr, spawned_stdout, p.wait())
 
 class Base(object):
     def __init__(self, path, spe=None):
         if s < 0 or s > 2:
             sys.stdout.write("Unexpected exit status %d\n" % s)
 
-if not use_subprocess:
-    import popen2
-    try:
-        popen2.Popen3
-    except AttributeError:
-        class PopenExecutor(Base):
-            def execute(self):
-                (tochild, fromchild, childerr) = os.popen3(self.command_str)
-                tochild.close()
-                self.stderr = childerr.read()
-                self.stdout = fromchild.read()
-                fromchild.close()
-                self.status = childerr.close()
-                if not self.status:
-                    self.status = 0
-                else:
-                    self.status = self.status >> 8
-    else:
-        class PopenExecutor(Base):
-            def execute(self):
-                p = popen2.Popen3(self.command_str, 1)
-                p.tochild.close()
-                self.stdout = p.fromchild.read()
-                self.stderr = p.childerr.read()
-                self.status = p.wait()
-                self.status = self.status >> 8
-else:
-    class PopenExecutor(Base):
-        # For an explanation of the following 'if ... else'
-        # and the 'allow_pipe_files' option, please check out the
-        # use_subprocess path in the definition of spawn_it() above.
-        if allow_pipe_files:
-            def execute(self):
-                # Create temporary files
-                import tempfile
-                tmp_stdout = tempfile.TemporaryFile(mode='w+t')
-                tmp_stderr = tempfile.TemporaryFile(mode='w+t')
-                # Start subprocess...
-                p = subprocess.Popen(self.command_str,
-                                     stdout=tmp_stdout,
-                                     stderr=tmp_stderr,
-                                     shell=True)
-                # ... and wait for it to finish.
-                self.status = p.wait()
-                
-                try:
-                    # Rewind to start of files
-                    tmp_stdout.seek(0)
-                    tmp_stderr.seek(0)
-                    # Read output
-                    self.stdout = tmp_stdout.read()
-                    self.stderr = tmp_stderr.read()
-                finally:
-                    # Remove temp files by closing them
-                    tmp_stdout.close()
-                    tmp_stderr.close()
-        else:        
-            def execute(self):
-                p = subprocess.Popen(self.command_str,
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE,
-                                     shell=True)
-                self.stdout = p.stdout.read()
-                self.stderr = p.stderr.read()
-                self.status = p.wait()
+class PopenExecutor(Base):
+    # For an explanation of the following 'if ... else'
+    # and the 'allow_pipe_files' option, please check out the
+    # definition of spawn_it() above.
+    if allow_pipe_files:
+        def execute(self):
+            # Create temporary files
+            import tempfile
+            tmp_stdout = tempfile.TemporaryFile(mode='w+t')
+            tmp_stderr = tempfile.TemporaryFile(mode='w+t')
+            # Start subprocess...
+            p = subprocess.Popen(self.command_str,
+                                 stdout=tmp_stdout,
+                                 stderr=tmp_stderr,
+                                 shell=True)
+            # ... and wait for it to finish.
+            self.status = p.wait()
+            
+            try:
+                # Rewind to start of files
+                tmp_stdout.seek(0)
+                tmp_stderr.seek(0)
+                # Read output
+                self.stdout = tmp_stdout.read()
+                self.stderr = tmp_stderr.read()
+            finally:
+                # Remove temp files by closing them
+                tmp_stdout.close()
+                tmp_stderr.close()
+    else:        
+        def execute(self):
+            p = subprocess.Popen(self.command_str,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE,
+                                 shell=True)
+            self.stdout = p.stdout.read()
+            self.stderr = p.stderr.read()
+            self.status = p.wait()
 
 class XML(PopenExecutor):
     def header(self, f):
         f.write('  <time>%.1f</time>\n' % self.total_time)
         f.write('  </results>\n')
 
-format_class = {
-    None        : SystemExecutor,
-    '--xml'     : XML,
-}
+if options.xml:
+    Test = XML
+else:
+    Test = SystemExecutor
 
-Test = format_class[format]
-
+# --- start processing ---
 if package:
 
     dir = {
     tests = [x for x in tests if x[0] != '#']
     tests = [x[:-1] for x in tests]
     tests = [x.strip() for x in tests]
-elif all and not qmtest:
+elif options.all and not qmtest:
     # Find all of the SCons functional tests in the local directory
     # tree.  This is anything under the 'src' subdirectory that ends
     # with 'Tests.py', or any Python script (*.py) under the 'test'
     tests.extend(find_py('test'))
     tests.sort()
 
+if not tests:
+    sys.stderr.write("""\
+runtest.py:  No tests were found.
+""")
+    sys.exit(1)
+
 if qmtest:
     if baseline:
         aegis_result_stream = 'scons_tdb.AegisBaselineStream'
     if python:
         qmtest_args.append('--context python="%s"' % python)
 
-    if outputfile:
-        if format == '--xml':
-            rsclass = 'scons_tdb.SConsXMLResultStream'
-        else:
-            rsclass = 'scons_tdb.AegisBatchStream'
-        qof = "r'" + outputfile + "'"
+    if options.xml:
+        rsclass = 'scons_tdb.SConsXMLResultStream'
+        qof = "r'" + options.xml + "'"
         rs = '--result-stream="%s(filename=%s)"' % (rsclass, qof)
         qmtest_args.append(rs)
 
 
 tests = [Test(t) for t in tests]
 
-class Unbuffered(object):
-    def __init__(self, file):
-        self.file = file
-        self.softspace = 0  ## backward compatibility; not supported in Py3k
-    def write(self, arg):
-        self.file.write(arg)
-        self.file.flush()
-    def __getattr__(self, attr):
-        return getattr(self.file, attr)
-
-sys.stdout = Unbuffered(sys.stdout)
-sys.stderr = Unbuffered(sys.stderr)
-
 if list_only:
     for t in tests:
         sys.stdout.write(t.path + "\n")
 
 total_start_time = time_func()
 total_num_tests = len(tests)
-for idx,t in enumerate(tests):
+tests_completed = 0
+
+def run_test(t, io_lock, async=True):
+    global tests_completed
+    header = ""
     command_args = ['-tt']
     if python3incompatibilities:
         command_args.append('-3')
     t.command_str = " ".join([escape(python)] + command_args)
     if printcommand:
         if print_progress:
-            sys.stdout.write("%d/%d (%.2f%s) %s\n" % (idx+1, total_num_tests,
-                                                      float(idx+1)*100.0/float(total_num_tests),
-                                                      '%',
-                                                      t.command_str))
+            tests_completed += 1
+            n = tests_completed # approx indicator of where we are
+            header += ("%d/%d (%.2f%s) %s\n" % (n, total_num_tests,
+                                                float(n)*100.0/float(total_num_tests),
+                                                '%',
+                                                t.command_str))
         else:
-            sys.stdout.write(t.command_str + "\n")
+            header += t.command_str + "\n"
+    if not suppress_stdout and not suppress_stderr:
+        sys.stdout.write(header)
     head, tail = os.path.split(t.abspath)
     if head:
         os.environ['PYTHON_SCRIPT_DIR'] = head
     test_start_time = time_func()
     if execute_tests:
         t.execute()
-        if not suppress_stdout and t.stdout:
-            print t.stdout
-        if not suppress_stderr and t.stderr:
-            print t.stderr
 
     t.test_time = time_func() - test_start_time
+    if io_lock:
+        io_lock.acquire()
+    if suppress_stdout or suppress_stderr:
+        sys.stdout.write(header)
+    if not suppress_stdout and t.stdout:
+        print t.stdout
+    if not suppress_stderr and t.stderr:
+        print t.stderr
     print_time_func("Test execution time: %.1f seconds\n", t.test_time)
+    if io_lock:
+        io_lock.release()
+
+class RunTest(threading.Thread):
+    def __init__(self, queue, io_lock):
+        threading.Thread.__init__(self)
+        self.queue = queue
+        self.io_lock = io_lock
+
+    def run(self):
+        while True:
+            t = self.queue.get()
+            run_test(t, io_lock, True)
+            self.queue.task_done()
+
+if jobs > 1 and threading_ok:
+    print "Running tests using %d jobs"%jobs
+    # Start worker threads
+    queue = Queue.Queue()
+    io_lock = threading.Lock()
+    for i in range(1, jobs):
+        t = RunTest(queue, io_lock)
+        t.daemon = True
+        t.start()
+    # Give tasks to workers
+    for t in tests:
+        queue.put(t)
+    queue.join()
+else:
+    # Run tests serially
+    if jobs > 1:
+        print "Ignoring -j%d option; no python threading module available."%jobs
+    for t in tests:
+        run_test(t, None, False)
+
+# --- all tests are complete by the time we get here ---
 if len(tests) > 0:
     tests[0].total_time = time_func() - total_start_time
     print_time_func("Total execution time for all tests: %.1f seconds\n", tests[0].total_time)
         paths = [x.path for x in no_result]
         sys.stdout.write("\t" + "\n\t".join(paths) + "\n")
 
-if outputfile:
-    if outputfile == '-':
+if options.xml:
+    if options.xml == '-':
         f = sys.stdout
     else:
-        f = open(outputfile, 'w')
+        f = open(options.xml, 'w')
     tests[0].header(f)
     #f.write("test_result = [\n")
     for t in tests:
         t.write(f)
     tests[0].footer(f)
     #f.write("];\n")
-    if outputfile != '-':
+    if options.xml != '-':
         f.close()
 
 if len(fail):
     - Added ability to run scripts/scons.py directly from source checkout
     - Hide deprecated --debug={dtree,stree,tree} from --help output
     - Error messages from option parser now include hints about valid choices
+    - Cleaned up some Python 1.5 and pre-2.3 code, so don't expect SCons
+      to run on anything less than Python 2.4 anymore
+    - Several fixes for runtest.py:
+      * exit with an error if no tests were found
+      * removed --noqmtest option - this behavior is by default
+      * replaced `-o FILE --xml` combination with `--xml FILE`
+      * changed `-o, --output FILE` option to capture stdout/stderr output
+        from runtest.py
 
   From Juan Lang:
     - Fix WiX Tool to use .wixobj rather than .wxiobj for compiler output
       * removed Aegis support from runtest.py. (#2872)
 
   From Gary Oberbrunner:
+    - Add -jN support to runtest.py to run tests in parallel
     - Add MSVC10 and MSVC11 support to get_output low-level bat script runner.
     - Fix MSVS solution generation for VS11, and fixed tests.
 
     "No such file or directory",
     "The system cannot find the file specified",
     "The system cannot find the path specified",
+    "Das System kann die angegebene Datei nicht finden",
 ]
 
 test.must_contain_any_line(test.stderr(), fail_strings)

test/Errors/execute-a-directory.py

 """
 
 unrecognized = """\
-'%s' is not recognized as an internal or external command,
+'.+' is not recognized as an internal or external command,
 operable program or batch file.
-scons: *** [%s] Error 1
+scons: \*\*\* \[%s\] Error 1
 """
 
 unspecified = """\
 The name specified is not recognized as an
 internal or external command, operable program or batch file.
-scons: *** [%s] Error 1
+scons: \*\*\* \[%s\] Error 1
 """
 
 cannot_execute = """\
-%s: cannot execute
-scons: *** [%s] Error %s
-"""
-
-Permission_denied = """\
-%s: Permission denied
-scons: *** [%s] Error %s
+(sh: )*.+: cannot execute
+scons: \*\*\* \[%s\] Error %s
 """
 
 permission_denied = """\
-%s: permission denied
-scons: *** [%s] Error %s
+.+: (p|P)ermission denied
+scons: \*\*\* \[%s\] Error %s
 """
 
 is_a_directory = """\
-%s: is a directory
-scons: *** [%s] Error %s
+.+: (i|I)s a directory
+scons: \*\*\* \[%s\] Error %s
 """
 
-Is_a_directory = """\
-%s: Is a directory
-scons: *** [%s] Error %s
+konnte_nicht_gefunden_werden = """\
+Der Befehl ".+" ist entweder falsch geschrieben oder
+konnte nicht gefunden werden.
+scons: \*\*\* \[%s\] Error %s
 """
 
 test.description_set("Incorrect STDERR:\n%s\n" % test.stderr())
 if os.name == 'nt':
     errs = [
         bad_command,
-        unrecognized % (test.workdir, 'f3'),
+        unrecognized % 'f3',
+        konnte_nicht_gefunden_werden % ('f3', 1),
         unspecified % 'f3'
     ]
-    test.fail_test(not test.stderr() in errs)
 elif sys.platform.find('sunos') != -1:
     errs = [
-        cannot_execute % ('sh: %s' % test.workdir, 'f3', 1),
+        cannot_execute % ('f3', 1),
     ]
-    test.fail_test(not test.stderr() in errs)
 else:
     errs = [
-        cannot_execute % (not_executable, 'f3', 126),
-        is_a_directory % (test.workdir, 'f3', 126),
-        Is_a_directory % (test.workdir, 'f3', 126),
-        Permission_denied % (test.workdir, 'f3', 126),
+        cannot_execute % ('f3', 126),
+        is_a_directory % ('f3', 126),
+        permission_denied % ('f3', 126),
     ]
-    test.must_contain_any_line(test.stderr(), errs)
+
+test.must_contain_any_line(test.stderr(), errs, find=TestSCons.search_re)
 
 test.pass_test()
 

test/Errors/non-executable-file.py

 """
 
 unrecognized = """\
-'%s' is not recognized as an internal or external command,
+'.+' is not recognized as an internal or external command,
 operable program or batch file.
-scons: *** [%s] Error 1
+scons: \*\*\* \[%s\] Error 1
 """
 
 unspecified = """\
 The name specified is not recognized as an
 internal or external command, operable program or batch file.
-scons: *** [%s] Error 1
+scons: \*\*\* \[%s\] Error 1
 """
 
 cannot_execute = """\
-%s: cannot execute
-scons: *** [%s] Error %s
-"""
-
-Permission_denied = """\
-%s: Permission denied
-scons: *** [%s] Error %s
+(sh: )*.+: cannot execute
+scons: \*\*\* \[%s\] Error %s
 """
 
 permission_denied = """\
-%s: permission denied
-scons: *** [%s] Error %s
+.+: (p|P)ermission denied
+scons: \*\*\* \[%s\] Error %s
+"""
+
+konnte_nicht_gefunden_werden = """\
+Der Befehl ".+" ist entweder falsch geschrieben oder
+konnte nicht gefunden werden.
+scons: \*\*\* \[%s\] Error %s
 """
 
 test.write('SConstruct', r"""
 if os.name == 'nt':
     errs = [
         bad_command,
-        unrecognized % (not_executable, 'f1'),
+        unrecognized % 'f1',
+        konnte_nicht_gefunden_werden % ('f1', 1),
         unspecified % 'f1'
     ]
-    test.fail_test(not test.stderr() in errs)
 elif sys.platform.find('sunos') != -1:
     errs = [
-        cannot_execute % ('sh: %s' % not_executable, 'f1', 1),
+        cannot_execute % ('f1', 1),
     ]
-    test.fail_test(not test.stderr() in errs)
 else:
     errs = [
-        cannot_execute % (not_executable, 'f1', 126),
-        Permission_denied % (not_executable, 'f1', 126),
-        permission_denied % (not_executable, 'f1', 126),
+        cannot_execute % ('f1', 126),
+        permission_denied % ('f1', 126),
     ]
-    test.must_contain_any_line(test.stderr(), errs)
+
+test.must_contain_any_line(test.stderr(), errs, find=TestSCons.search_re)
 
 test.pass_test()
 

test/Errors/nonexistent-executable.py

          stderr = None,
          status = 2)
 
-bad_command = """\
-Bad command or file name
-"""
-
-unrecognized = """\
-'%s' is not recognized as an internal or external command,
-operable program or batch file.
-scons: *** [%s] Error 1
-"""
-
-unspecified = """\
-The name specified is not recognized as an
-internal or external command, operable program or batch file.
-scons: *** [%s] Error 1
-"""
-
-not_found_1_space = """\
-sh: %s: not found
-scons: *** [%s] Error %s
-"""
-
-not_found_2_spaces = """\
-sh: %s:  not found
-scons: *** [%s] Error %s
-"""
-
-No_such = """\
-%s: No such file or directory
-scons: *** [%s] Error %s
-"""
+bad_command = """Bad command or file name"""
+unrecognized = r"""'.+' is not recognized as an internal or external command,\s+operable program or batch file.\sscons: \*\*\* \[%s\] Error 1"""
+unspecified = r"""The name specified is not recognized as an\s+internal or external command, operable program or batch file.\s+scons: \*\*\* \[%s\] Error 1"""
+not_found_space = r"""sh: (\d: )*.+: \s*not found\s+scons: \*\*\* \[%s\] Error %s"""
+No_such = r""".+: No such file or directory\s+scons: \*\*\* \[%s\] Error %s"""
+konnte_nicht_gefunden_werden = r"""Der Befehl ".+" ist entweder falsch geschrieben oder\s+konnte nicht gefunden werden.\s+scons: \*\*\* \[%s\] Error %s"""
 
 test.description_set("Incorrect STDERR:\n%s\n" % test.stderr())
 if os.name == 'nt':
     errs = [
         bad_command,
-        unrecognized % (no_such_file, 'f1'),
+        unrecognized % 'f1',
+        konnte_nicht_gefunden_werden % ('f1', 1),
         unspecified % 'f1'
     ]
-    test.fail_test(not test.stderr() in errs)
 elif sys.platform.find('sunos') != -1:
     errs = [
-        not_found_1_space % (no_such_file, 'f1', 1),
+        not_found_space % ('f1', 1),
     ]
-    test.fail_test(not test.stderr() in errs)
 else:
     errs = [
-        not_found_1_space % (no_such_file, 'f1', 1),
-        not_found_2_spaces % (no_such_file, 'f1', 1),
-        not_found_1_space % (no_such_file, 'f1', 127),
-        No_such % (no_such_file, 'f1', 127),
+        not_found_space % ('f1', 1),
+        not_found_space % ('f1', 127),
+        No_such % ('f1', 127),
     ]
-    test.must_contain_any_line(test.stderr(), errs)
+
+test.must_contain_any_line(test.stderr(), errs, find=TestSCons.search_re)
 
 test.pass_test()
 
 
 import sys
 if sys.platform == 'win32':
-    expect = """\
-scons: *** Error 1
-scons: *** Error 2
-scons: *** nonexistent.in/*.*: The system cannot find the path specified
-"""
+    expect = r"""scons: \*\*\* Error 1
+scons: \*\*\* Error 2
+scons: \*\*\* nonexistent.in/\*\.\*: (The system cannot find the path specified|Das System kann den angegebenen Pfad nicht finden)"""
 else:
-    expect = """\
-scons: *** Error 1
-scons: *** Error 2
-scons: *** nonexistent.in: No such file or directory
-"""
+    expect = r"""scons: \*\*\* Error 1
+scons: \*\*\* Error 2
+scons: \*\*\* nonexistent\.in: No such file or directory"""
 
-test.run(arguments = '.', stderr=expect)
+test.run(arguments = '.', stdout = None, stderr = None)
+
+test.must_contain_all_lines(test.stderr(), expect.splitlines(), find=TestSCons.search_re)
 
 test.must_match('a.out', "a.in\n")
 test.must_match('b.out', "b.in\n")

test/Install/Install.py

 expect =  [
     "Permission denied",
     "The process cannot access the file because it is being used by another process",
+    "Der Prozess kann nicht auf die Datei zugreifen, da sie von einem anderen Prozess verwendet wird",
 ]
 
 test.run(chdir = 'work', arguments = f1_out, stderr=None, status=2)

test/Interactive/shell.py

 scons.send("\n")
 
 if sys.platform == 'win32':
-    no_such_error = "'no_such_command' is not recognized as an internal or external command,\noperable program or batch file."
+    no_such_error = r"('no_such_command' is not recognized as an internal or external command,\s+operable program or batch file\.|Der Befehl \"no_such_command\" ist entweder falsch geschrieben oder\s+konnte nicht gefunden werden\.)"
 else:
     no_such_error = 'scons: no_such_command: No such file or directory'
 
 expect_stdout = """\
-scons>>> Copy("foo.out", "foo.in")
-Touch("1")
+scons>>> Copy\("foo.out", "foo.in"\)
+Touch\("1"\)
 scons>>> hello from shell_command.py
-scons>>> !%(_python_)s %(_shell_command_py_)s
+scons>>> ![^"]+ ".*"
 hello from shell_command.py
 scons>>> hello from shell_command.py
-scons>>> shell %(_python_)s %(_shell_command_py_)s
+scons>>> shell [^"]+ ".*"
 hello from shell_command.py
 scons>>> hello from shell_command.py
-scons>>> sh %(_python_)s %(_shell_command_py_)s
+scons>>> sh [^"]+ ".*"
 hello from shell_command.py
 scons>>> %(no_such_error)s
 scons>>> !no_such_command arg1 arg2
 scons>>> 
 """ % locals()
 
-test.finish(scons, stdout = expect_stdout)
+test.finish(scons, stdout = None)
 
-
+test.must_contain_all_lines(test.stdout(), expect_stdout.splitlines(), find=TestSCons.search_re)
 
 test.pass_test()
 
 
 __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
-import sys
 import TestSCons
 
-if sys.platform == 'win32':
-    _obj = '.obj'
-else:
-    _obj = '.o'
+_obj = TestSCons._obj
 
 test = TestSCons.TestSCons()
 

test/Repository/StaticLibrary.py

 __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 import os.path
-import sys
 import TestSCons
 
-if sys.platform == 'win32':
-    _obj = '.obj'
-    _exe = '.exe'
-else:
-    _obj = '.o'
-    _exe = ''
-
-
+_obj = TestSCons._obj
+_exe = TestSCons._exe
 
 test = TestSCons.TestSCons()
 

test/scons-time/func/file.py

 set key bottom left
 set label 3 "label 1.5" at 0.5,0.5 right
 set label 4 "label 1.6" at 0.6,0.4 right
-plot '-' title "Startup" with lines lt 1, \
-     '-' notitle with lines lt 7, \
-     '-' title "label 1.5" with lines lt 7, \
+plot '-' title "Startup" with lines lt 1, \\
+     '-' notitle with lines lt 7, \\
+     '-' title "label 1.5" with lines lt 7, \\
      '-' title "label 1.6" with lines lt 7
 # Startup
 1 0.000
-2 0.000
+2 0.\d*
 e
 1.4 0
 1.4 1
 e
 """
 
-test.run(arguments = 'func --file st2.conf --fmt gnuplot', stdout = expect2)
+test.run(arguments = 'func --file st2.conf --fmt gnuplot')
 
+test.must_contain_exactly_lines(test.stdout(), expect2, find=TestSCons_time.search_re_in_list)
 
 test.pass_test()