Commits

Anonymous committed 2064312

Fix the tests of runtest.py now that QMTest is no longer being used
by default. Fix use of subprocess in Python 2.4+ and exit statuses
of popen'ed scripts in earlier versions of Python. Support the ability
to execute a directory's tests by naming the directory as a command-
line argument.

Comments (0)

Files changed (16)

                 self.status = childerr.close()
                 if not self.status:
                     self.status = 0
+                else:
+                    self.status = self.status >> 8
     else:
         class PopenExecutor(Base):
             def execute(self):
                 self.stdout = p.fromchild.read()
                 self.stderr = p.childerr.read()
                 self.status = p.wait()
+                self.status = self.status >> 8
 else:
     class PopenExecutor(Base):
         def execute(self):
-            p = subprocess.Popen(self.command_str, shell=True)
-            p.stdin.close()
+            p = subprocess.Popen(self.command_str,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE,
+                                 shell=True)
             self.stdout = p.stdout.read()
-            self.stdout = p.stderr.read()
+            self.stderr = p.stderr.read()
             self.status = p.wait()
 
 class Aegis(SystemExecutor):
 
 tests = []
 
+def find_Tests_py(tdict, dirname, names):
+    for n in filter(lambda n: n[-8:] == "Tests.py", names):
+        tdict[os.path.join(dirname, n)] = 1
+
+def find_py(tdict, dirname, names):
+    tests = filter(lambda n: n[-3:] == ".py", names)
+    try:
+        excludes = open(os.path.join(dirname,".exclude_tests")).readlines()
+    except (OSError, IOError):
+        pass
+    else:
+        for exclude in excludes:
+            exclude = string.split(exclude, '#' , 1)[0]
+            exclude = string.strip(exclude)
+            if not exclude: continue
+            tests = filter(lambda n, ex = exclude: n != ex, tests)
+    for n in tests:
+        tdict[os.path.join(dirname, n)] = 1
+
 if args:
     if spe:
         for a in args:
                         break
     else:
         for a in args:
-            tests.extend(glob.glob(a))
+            for path in glob.glob(a):
+                if os.path.isdir(path):
+                    tdict = {}
+                    if path[:3] == 'src':
+                        os.path.walk(path, find_Tests_py, tdict)
+                    elif path[:4] == 'test':
+                        os.path.walk(path, find_py, tdict)
+                    t = tdict.keys()
+                    t.sort()
+                    tests.extend(t)
+                else:
+                    tests.append(path)
 elif testlistfile:
     tests = open(testlistfile, 'r').readlines()
     tests = filter(lambda x: x[0] != '#', tests)
     # by the Aegis packaging build to make sure that we're building
     # things correctly.)
     tdict = {}
-
-    def find_Tests_py(tdict, dirname, names):
-        for n in filter(lambda n: n[-8:] == "Tests.py", names):
-            tdict[os.path.join(dirname, n)] = 1
     os.path.walk('src', find_Tests_py, tdict)
-
-    def find_py(tdict, dirname, names):
-        tests = filter(lambda n: n[-3:] == ".py", names)
-        try:
-            excludes = open(os.path.join(dirname,".exclude_tests")).readlines()
-        except (OSError, IOError):
-            pass
-        else:
-            for exclude in excludes:
-                exclude = string.split(exclude, '#' , 1)[0]
-                exclude = string.strip(exclude)
-                if not exclude: continue
-                tests = filter(lambda n, ex = exclude: n != ex, tests)
-        for n in tests:
-            tdict[os.path.join(dirname, n)] = 1
     os.path.walk('test', find_py, tdict)
-
     if format == '--aegis' and aegis:
         cmd = "aegis -list -unf pf 2>/dev/null"
         for line in os.popen(cmd, "r").readlines():
         return getattr(self.file, attr)
 
 sys.stdout = Unbuffered(sys.stdout)
+sys.stderr = Unbuffered(sys.stderr)
 
 if list_only:
     for t in tests:

test/runtest/aegis/batch-output.py

 
 test.write_passing_test(['test', 'pass.py'])
 
-test.run(arguments = '-o aegis.out --aegis test', status=1)
+expect_stderr = """\
+FAILING TEST STDERR
+NO RESULT TEST STDERR
+PASSING TEST STDERR
+"""
+
+test.run(arguments = '-o aegis.out --aegis test', stderr=expect_stderr)
 
 expect = """\
 test_result = [

test/runtest/baseline/combined.py

 
 import TestRuntest
 
+python = TestRuntest.python
 test_fail_py = os.path.join('test', 'fail.py')
 test_no_result_py = os.path.join('test', 'no_result.py')
 test_pass_py = os.path.join('test', 'pass.py')
 
 test.write_passing_test(['test', 'pass.py'])
 
-# NOTE:  The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both
-# have spaces at the end.
+expect_stdout = """\
+%(python)s -tt test/fail.py
+FAILING TEST STDOUT
+%(python)s -tt test/no_result.py
+NO RESULT TEST STDOUT
+%(python)s -tt test/pass.py
+PASSING TEST STDOUT
 
-expect = r"""qmtest run --output baseline.qmr --format none --result-stream="scons_tdb.AegisBaselineStream" test
---- TEST RESULTS -------------------------------------------------------------
+Failed the following test:
+\ttest/fail.py
 
-  %(test_fail_py)s                                  : FAIL    
-
-    FAILING TEST STDOUT
-
-    FAILING TEST STDERR
-
-  %(test_no_result_py)s                             : NO_RESULT
-
-    NO RESULT TEST STDOUT
-
-    NO RESULT TEST STDERR
-
-  %(test_pass_py)s                                  : PASS    
-
---- TESTS WITH UNEXPECTED OUTCOMES -------------------------------------------
-
-  %(test_no_result_py)s                             : NO_RESULT
-
-  %(test_pass_py)s                                  : PASS    
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       1 ( 33%%) tests as expected
-       1 ( 33%%) tests unexpected PASS
-       1 ( 33%%) tests unexpected NO_RESULT
+NO RESULT from the following test:
+\ttest/no_result.py
 """ % locals()
 
-test.run(arguments = '-b . test', status = 1, stdout = expect)
+expect_stderr = """\
+FAILING TEST STDERR
+NO RESULT TEST STDERR
+PASSING TEST STDERR
+"""
+
+test.run(arguments='-b . test',
+         status=1,
+         stdout=expect_stdout,
+         stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/baseline/fail.py

 
 import TestRuntest
 
+python = TestRuntest.python
+
 test = TestRuntest.TestRuntest()
 
 test.subdir('test')
 
 test.write_failing_test(['test', 'fail.py'])
 
-# NOTE:  The "test/fail.py   : FAIL" line has spaces at the end.
+expect_stdout = """\
+%(python)s -tt test/fail.py
+FAILING TEST STDOUT
+""" % locals()
 
-expect = r"""qmtest run --output baseline.qmr --format none --result-stream="scons_tdb.AegisBaselineStream" test/fail.py
---- TEST RESULTS -------------------------------------------------------------
-
-  test/fail.py                                  : FAIL    
-
-    FAILING TEST STDOUT
-
-    FAILING TEST STDERR
-
---- TESTS WITH UNEXPECTED OUTCOMES -------------------------------------------
-
-  None.
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       1 (100%) tests as expected
+expect_stderr = """\
+FAILING TEST STDERR
 """
 
-test.run(arguments = '-b . test/fail.py', status = 1, stdout = expect)
+test.run(arguments='-b . test/fail.py',
+         status=1,
+         stdout=expect_stdout,
+         stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/baseline/no_result.py

 
 import TestRuntest
 
+python = TestRuntest.python
+
 test = TestRuntest.TestRuntest()
 
 test.subdir('test')
 
 test.write_no_result_test(['test', 'no_result.py'])
 
-expect = r"""qmtest run --output baseline.qmr --format none --result-stream="scons_tdb.AegisBaselineStream" test/no_result.py
---- TEST RESULTS -------------------------------------------------------------
+expect_stdout = """\
+%(python)s -tt test/no_result.py
+NO RESULT TEST STDOUT
+""" % locals()
 
-  test/no_result.py                             : NO_RESULT
-
-    NO RESULT TEST STDOUT
-
-    NO RESULT TEST STDERR
-
---- TESTS WITH UNEXPECTED OUTCOMES -------------------------------------------
-
-  test/no_result.py                             : NO_RESULT
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       1 (100%) tests unexpected NO_RESULT
+expect_stderr = """\
+NO RESULT TEST STDERR
 """
 
-test.run(arguments = '-b . test/no_result.py', status = 1, stdout = expect)
+test.run(arguments='-b . test/no_result.py',
+         status=2,
+         stdout=expect_stdout,
+         stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/baseline/pass.py

 
 import TestRuntest
 
+python = TestRuntest.python
+
 test = TestRuntest.TestRuntest()
 
 test.subdir('test')
 
 test.write_passing_test(['test', 'pass.py'])
 
-# NOTE:  The "test/pass.py   : PASS" line has spaces at the end.
+expect_stdout = """\
+%(python)s -tt test/pass.py
+PASSING TEST STDOUT
+""" % locals()
 
-expect = r"""qmtest run --output baseline.qmr --format none --result-stream="scons_tdb.AegisBaselineStream" test/pass.py
---- TEST RESULTS -------------------------------------------------------------
-
-  test/pass.py                                  : PASS    
-
---- TESTS WITH UNEXPECTED OUTCOMES -------------------------------------------
-
-  test/pass.py                                  : PASS    
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       1 (100%) tests unexpected PASS
+expect_stderr = """\
+PASSING TEST STDERR
 """
 
-test.run(arguments = '-b . test/pass.py', stdout = expect)
+test.run(arguments='-b . test',
+         stdout=expect_stdout,
+         stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/fallback.py

 """ % locals()
 
 expect_stderr = """\
-Warning:  found neither qmtest nor qmtest.py on $PATH;
-\tassuming --noqmtest option.
 FAILING TEST STDERR
 NO RESULT TEST STDERR
 PASSING TEST STDERR

test/runtest/print_time.py

 import TestCmd
 import TestRuntest
 
+python = TestRuntest.python
 test_fail_py = re.escape(os.path.join('test', 'fail.py'))
 test_no_result_py = re.escape(os.path.join('test', 'no_result.py'))
 test_pass_py = re.escape(os.path.join('test', 'pass.py'))
 
 test.write_passing_test(['test', 'pass.py'])
 
-# NOTE:  The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both
-# have spaces at the end.
+expect_stdout = """\
+%(python)s -tt test/fail.py
+FAILING TEST STDOUT
+Test execution time: \\d+.\\d seconds
+%(python)s -tt test/no_result.py
+NO RESULT TEST STDOUT
+Test execution time: \\d+.\\d seconds
+%(python)s -tt test/pass.py
+PASSING TEST STDOUT
+Test execution time: \\d+.\\d seconds
+Total execution time for all tests: \\d+.\\d seconds
 
-expect = r"""qmtest run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream\(print_time='1'\)" test
---- TEST RESULTS -------------------------------------------------------------
+Failed the following test:
+\ttest/fail.py
 
-  %(test_fail_py)s                                  : FAIL    
-
-    FAILING TEST STDOUT
-
-    FAILING TEST STDERR
-
-    Total execution time: \d+\.\d+ seconds
-
-  %(test_no_result_py)s                             : NO_RESULT
-
-    NO RESULT TEST STDOUT
-
-    NO RESULT TEST STDERR
-
-    Total execution time: \d+\.\d+ seconds
-
-  %(test_pass_py)s                                  : PASS    
-
-    Total execution time: \d+\.\d+ seconds
-
---- TESTS THAT DID NOT PASS --------------------------------------------------
-
-  %(test_fail_py)s                                  : FAIL    
-
-  %(test_no_result_py)s                             : NO_RESULT
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       3        tests total
-
-       1 \( 33%%\) tests PASS
-       1 \( 33%%\) tests FAIL
-       1 \( 33%%\) tests NO_RESULT
+NO RESULT from the following test:
+\ttest/no_result.py
 """ % locals()
 
-test.run(arguments = '-t test', status = 1, stdout = expect)
+expect_stderr = """\
+FAILING TEST STDERR
+NO RESULT TEST STDERR
+PASSING TEST STDERR
+"""
+
+test.run(arguments='-t test',
+         status=1,
+         stdout=expect_stdout,
+         stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/python.py

 
 test.write_passing_test(['test', 'pass.py'])
 
-# NOTE:  The "test/pass.py : PASS" line has spaces at the end.
-
-expect = r"""qmtest run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" --context python="%(mypython)s" test
---- TEST RESULTS -------------------------------------------------------------
-
-  %(test_pass_py)s                                  : PASS    
-
---- TESTS THAT DID NOT PASS --------------------------------------------------
-
-  None.
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       1        tests total
-
-       1 (100%%) tests PASS
+expect_stdout = """\
+%(mypython)s -tt test/pass.py
+PASSING TEST STDOUT
 """ % locals()
 
-test.run(arguments = ['-P', mypython, 'test'], stdout = expect)
+expect_stderr = """\
+PASSING TEST STDERR
+"""
+
+test.run(arguments=['-P', mypython, 'test'],
+         stdout=expect_stdout,
+         stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/simple/combined.py

 
 test = TestRuntest.TestRuntest()
 
+python = TestRuntest.python
 test_fail_py = os.path.join('test', 'fail.py')
 test_no_result_py = os.path.join('test', 'no_result.py')
 test_pass_py = os.path.join('test', 'pass.py')
 
 test.write_passing_test(['test', 'pass.py'])
 
-# NOTE:  The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both
-# have spaces at the end.
+expect_stdout = """\
+%(python)s -tt test/fail.py
+FAILING TEST STDOUT
+%(python)s -tt test/no_result.py
+NO RESULT TEST STDOUT
+%(python)s -tt test/pass.py
+PASSING TEST STDOUT
 
-expect = r"""qmtest run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" test
---- TEST RESULTS -------------------------------------------------------------
+Failed the following test:
+\ttest/fail.py
 
-  %(test_fail_py)s                                  : FAIL    
-
-    FAILING TEST STDOUT
-
-    FAILING TEST STDERR
-
-  %(test_no_result_py)s                             : NO_RESULT
-
-    NO RESULT TEST STDOUT
-
-    NO RESULT TEST STDERR
-
-  %(test_pass_py)s                                  : PASS    
-
---- TESTS THAT DID NOT PASS --------------------------------------------------
-
-  %(test_fail_py)s                                  : FAIL    
-
-  %(test_no_result_py)s                             : NO_RESULT
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       3        tests total
-
-       1 ( 33%%) tests PASS
-       1 ( 33%%) tests FAIL
-       1 ( 33%%) tests NO_RESULT
+NO RESULT from the following test:
+\ttest/no_result.py
 """ % locals()
 
-test.run(arguments = 'test', status = 1, stdout = expect)
+expect_stderr = """\
+FAILING TEST STDERR
+NO RESULT TEST STDERR
+PASSING TEST STDERR
+"""
+
+test.run(arguments='test',
+         status=1,
+         stdout=expect_stdout,
+         stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/simple/fail.py

 
 import TestRuntest
 
+python = TestRuntest.python
+
 test = TestRuntest.TestRuntest()
 
 test.subdir('test')
 
 test.write_failing_test(['test', 'fail.py'])
 
-# NOTE:  The "test/fail.py   : FAIL" line has spaces at the end.
+expect_stdout = """\
+%(python)s -tt test/fail.py
+FAILING TEST STDOUT
+""" % locals()
 
-expect = r"""qmtest run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" test/fail.py
---- TEST RESULTS -------------------------------------------------------------
-
-  test/fail.py                                  : FAIL    
-
-    FAILING TEST STDOUT
-
-    FAILING TEST STDERR
-
---- TESTS THAT DID NOT PASS --------------------------------------------------
-
-  test/fail.py                                  : FAIL    
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       1        tests total
-
-       1 (100%) tests FAIL
+expect_stderr = """\
+FAILING TEST STDERR
 """
 
-test.run(arguments = 'test/fail.py', status = 1, stdout = expect)
+test.run(arguments='test/fail.py',
+         status=1,
+         stdout=expect_stdout,
+         stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/simple/no_result.py

 
 import TestRuntest
 
+python = TestRuntest.python
+
 test = TestRuntest.TestRuntest()
 
 test.subdir('test')
 
 test.write_no_result_test(['test', 'no_result.py'])
 
-expect = r"""qmtest run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" test/no_result.py
---- TEST RESULTS -------------------------------------------------------------
+expect_stdout = """\
+%(python)s -tt test/no_result.py
+NO RESULT TEST STDOUT
+""" % locals()
 
-  test/no_result.py                             : NO_RESULT
-
-    NO RESULT TEST STDOUT
-
-    NO RESULT TEST STDERR
-
---- TESTS THAT DID NOT PASS --------------------------------------------------
-
-  test/no_result.py                             : NO_RESULT
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       1        tests total
-
-       1 (100%) tests NO_RESULT
+expect_stderr = """\
+NO RESULT TEST STDERR
 """
 
-test.run(arguments = 'test/no_result.py', status = 1, stdout = expect)
+test.run(arguments='test/no_result.py',
+         status=2,
+         stdout=expect_stdout,
+         stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/simple/pass.py

 
 import TestRuntest
 
+python = TestRuntest.python
+
 test = TestRuntest.TestRuntest()
 
 test.subdir('test')
 
 test.write_passing_test(['test', 'pass.py'])
 
-# NOTE:  The "test/pass.py   : PASS" line has spaces at the end.
+expect_stdout = """\
+%(python)s -tt test/pass.py
+PASSING TEST STDOUT
+""" % locals()
 
-expect = r"""qmtest run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" test/pass.py
---- TEST RESULTS -------------------------------------------------------------
-
-  test/pass.py                                  : PASS    
-
---- TESTS THAT DID NOT PASS --------------------------------------------------
-
-  None.
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       1        tests total
-
-       1 (100%) tests PASS
+expect_stderr = """\
+PASSING TEST STDERR
 """
 
-test.run(arguments = 'test/pass.py', stdout = expect)
+test.run(arguments='test/pass.py', stdout=expect_stdout, stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/src.py

 
 import TestRuntest
 
-test = TestRuntest.TestRuntest(verbose=1)
+test = TestRuntest.TestRuntest()
 
 test.subdir(['src'],
             ['src', 'suite'])
 
+python = TestRuntest.python
 src_passTests_py = os.path.join('src', 'passTests.py')
 src_suite_passTests_py = os.path.join('src', 'suite', 'passTests.py')
 
 
 test.write_passing_test(['src', 'suite', 'passTests.py'])
 
-# NOTE:  The "test/pass.py : PASS" and "test/passTests.py : PASS" lines
-# both have spaces at the end.
-
-expect = r"""qmtest run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" src
---- TEST RESULTS -------------------------------------------------------------
-
-  %(src_passTests_py)s                              : PASS    
-
-  %(src_suite_passTests_py)s                        : PASS    
-
---- TESTS THAT DID NOT PASS --------------------------------------------------
-
-  None.
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       2        tests total
-
-       2 (100%%) tests PASS
+expect_stdout = """\
+%(python)s -tt src/passTests.py
+PASSING TEST STDOUT
+%(python)s -tt src/suite/passTests.py
+PASSING TEST STDOUT
 """ % locals()
 
-test.run(arguments = 'src', stdout = expect)
+expect_stderr = """\
+PASSING TEST STDERR
+PASSING TEST STDERR
+""" % locals()
+
+test.run(arguments='src', stdout=expect_stdout, stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/testlistfile.py

 
 import TestRuntest
 
+python = TestRuntest.python
 test_fail_py = os.path.join('test', 'fail.py')
 test_no_result_py = os.path.join('test', 'no_result.py')
 test_pass_py = os.path.join('test', 'pass.py')
 %(test_pass_py)s
 """ % locals())
 
-# NOTE:  The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both
-# have spaces at the end.
-
-expect = """qmtest run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" %(test_pass_py)s
---- TEST RESULTS -------------------------------------------------------------
-
-  %(test_pass_py)s                                  : PASS    
-
---- TESTS THAT DID NOT PASS --------------------------------------------------
-
-  None.
-
-
---- STATISTICS ---------------------------------------------------------------
-
-       1        tests total
-
-       1 (100%%) tests PASS
+expect_stdout = """\
+%(python)s -tt test/pass.py
+PASSING TEST STDOUT
 """ % locals()
 
-test.run(arguments = '-f t.txt', stdout = expect)
+expect_stderr = """\
+PASSING TEST STDERR
+"""
+
+test.run(arguments='-f t.txt', stdout=expect_stdout, stderr=expect_stderr)
 
 test.pass_test()
 

test/runtest/xml/output.py

 import re
 import sys
 
+import TestCmd
 import TestRuntest
 
-test = TestRuntest.TestRuntest()
+test = TestRuntest.TestRuntest(match = TestCmd.match_re)
 
+python = TestRuntest.python
 test_fail_py = re.escape(os.path.join('test', 'fail.py'))
 test_no_result_py = re.escape(os.path.join('test', 'no_result.py'))
 test_pass_py = re.escape(os.path.join('test', 'pass.py'))
 
 test.write_passing_test(['test', 'pass.py'])
 
-test.run(arguments = '-o xml.out --xml test', status = 1)
+test.run(arguments = '-o xml.out --xml test', status=1)
 
-expect_engine = """\
-<annotation key="scons_test\\.engine">
- __build__='D456'
- __buildsys__='another_fake_system'
- __date__='Dec 31 1999'
- __developer__='John Doe'
- __version__='4\\.5\\.6'
-</annotation>
-"""
-
-expect_script = """\
-<annotation key="scons_test\\.script">
- __build__='D123'
- __buildsys__='fake_system'
- __date__='Jan 1 1970'
- __developer__='Anonymous'
- __version__='1\\.2\\.3'
-</annotation>
-"""
-
-# The actual values printed for sys and os.environ will be completely
-# dependent on the local values.  Don't bother trying to match, just
-# look to see if the opening tag exists.
-
-expect_sys = """\
-<annotation key="scons_test\\.sys">
-"""
-
-expect_os_environ = """\
-<annotation key="scons_test\\.os\\.environ">
-"""
-
-expect_fail = """\
- <result id="%(test_fail_py)s" kind="test" outcome="FAIL">
-  <annotation name="Test\\.exit_code">
-   &quot;1&quot;
-  </annotation>
-  <annotation name="Test\\.stderr">
-   &quot;&lt;pre&gt;FAILING TEST STDERR%(cr)s
-&lt;/pre&gt;&quot;
-  </annotation>
-  <annotation name="Test\\.stdout">
-   &quot;&lt;pre&gt;FAILING TEST STDOUT%(cr)s
-&lt;/pre&gt;&quot;
-  </annotation>
-  <annotation name="qmtest\\.cause">
-   &quot;Non-zero exit_code\\.&quot;
-  </annotation>
-  <annotation name="qmtest\\.end_time">
-   &quot;[\\d.]+&quot;
-  </annotation>
-  <annotation name="qmtest\\.start_time">
-   &quot;[\\d.]+&quot;
-  </annotation>
-  <annotation name="qmtest\\.target">
-   &quot;local&quot;
-  </annotation>
- </result>
+expect = """\
+  <results>
+    <test>
+      <file_name>test/fail.py</file_name>
+      <command_line>%(python)s -tt test/fail.py</command_line>
+      <exit_status>1</exit_status>
+      <stdout>FAILING TEST STDOUT
+</stdout>
+      <stderr>FAILING TEST STDERR
+</stderr>
+      <time>\\d+\.\d</time>
+    </test>
+    <test>
+      <file_name>test/no_result.py</file_name>
+      <command_line>%(python)s -tt test/no_result.py</command_line>
+      <exit_status>2</exit_status>
+      <stdout>NO RESULT TEST STDOUT
+</stdout>
+      <stderr>NO RESULT TEST STDERR
+</stderr>
+      <time>\\d+\.\d</time>
+    </test>
+    <test>
+      <file_name>test/pass.py</file_name>
+      <command_line>%(python)s -tt test/pass.py</command_line>
+      <exit_status>0</exit_status>
+      <stdout>PASSING TEST STDOUT
+</stdout>
+      <stderr>PASSING TEST STDERR
+</stderr>
+      <time>\\d+\.\d</time>
+    </test>
+  <time>\\d+\.\d</time>
+  </results>
 """ % locals()
 
-expect_no_result = """\
- <result id="%(test_no_result_py)s" kind="test" outcome="FAIL">
-  <annotation name="Test.exit_code">
-   &quot;2&quot;
-  </annotation>
-  <annotation name="Test\\.stderr">
-   &quot;&lt;pre&gt;NO RESULT TEST STDERR%(cr)s
-&lt;/pre&gt;&quot;
-  </annotation>
-  <annotation name="Test\\.stdout">
-   &quot;&lt;pre&gt;NO RESULT TEST STDOUT%(cr)s
-&lt;/pre&gt;&quot;
-  </annotation>
-  <annotation name="qmtest\\.cause">
-   &quot;Non-zero exit_code\\.&quot;
-  </annotation>
-  <annotation name="qmtest\\.end_time">
-   &quot;[\\d.]+&quot;
-  </annotation>
-  <annotation name="qmtest\\.start_time">
-   &quot;[\\d.]+&quot;
-  </annotation>
-  <annotation name="qmtest\\.target">
-   &quot;local&quot;
-  </annotation>
- </result>
-""" % locals()
-
-expect_pass = """\
- <result id="%(test_pass_py)s" kind="test" outcome="PASS">
-  <annotation name="qmtest\\.end_time">
-   &quot;[\\d.]+&quot;
-  </annotation>
-  <annotation name="qmtest\\.start_time">
-   &quot;[\\d.]+&quot;
-  </annotation>
-  <annotation name="qmtest\\.target">
-   &quot;local&quot;
-  </annotation>
- </result>
-""" % locals()
-
-xml_out = test.read('xml.out', 'r')
-
-expect = [
-    expect_engine,
-    expect_script,
-    expect_sys,
-    expect_os_environ,
-    expect_fail,
-    expect_no_result,
-    expect_pass,
-]
-
-non_matches = []
-
-for e in expect:
-    if not re.search(e, xml_out):
-        non_matches.append(e)
-
-if non_matches:
-    for n in non_matches:
-        print "DID NOT MATCH " + '='*60
-        print n
-    print "ACTUAL XML OUTPUT " + '='*60
-    print xml_out
-    test.fail_test()
+test.must_match('xml.out', expect)
 
 test.pass_test()