Commits

Matt Oswald committed c73138c

refactored the timed test runner to account for abandoned long-running tests that end before the rest of the tests
Check is now passed around as a shared_ptr to make sure any abandoned long-running tests can still access it without exploding
TestRunner is no longer a class (not necessary)
RunTests returns the count of failed tests, not the negative count
many tests for the RunTests function

  • Participants
  • Parent commits 5326d77

Comments (0)

Files changed (18)

Tests/UnitTests/Attributes.cpp

     attributes.insert(std::make_pair("Skip", "Testing skip."));
 
     xUnitpp::TestCollection collection;
-    xUnitpp::Check check;
+    auto check = std::make_shared<xUnitpp::Check>();
     xUnitpp::TestCollection::Register reg(collection, []() { SkippedTest().RunTest(); },
         "SkippedTest", "Attributes", attributes, -1, __FILE__, __LINE__, check);
 
-    xUnitpp::TestRunner local(emptyReporter);
-    local.RunTests([](const xUnitpp::TestDetails &) { return true; },
+    xUnitpp::RunTests(emptyReporter, [](const xUnitpp::TestDetails &) { return true; },
         collection.Tests(), xUnitpp::Time::Duration::zero(), 0);
 }
 

Tests/UnitTests/Helpers/OutputRecord.cpp

+#include "OutputRecord.h"
+#include "xUnit++/LineInfo.h"
+#include "xUnit++/TestDetails.h"
+
+namespace xUnitpp { namespace Tests {
+
+void OutputRecord::ReportStart(const TestDetails &testDetails)
+{
+    std::lock_guard<std::mutex> guard(lock);
+    orderedTestList.push_back(testDetails);
+}
+
+void OutputRecord::ReportFailure(const TestDetails &testDetails, const std::string &msg, const LineInfo &lineInfo)
+{
+    std::lock_guard<std::mutex> guard(lock);
+    failures.push_back(std::make_tuple(testDetails, msg, lineInfo));
+}
+
+void OutputRecord::ReportSkip(const TestDetails &testDetails, const std::string &reason)
+{
+    std::lock_guard<std::mutex> guard(lock);
+    skips.push_back(std::make_tuple(testDetails, reason));
+}
+
+void OutputRecord::ReportFinish(const TestDetails &testDetails, Time::Duration timeTaken)
+{
+    std::lock_guard<std::mutex> guard(lock);
+    finishedTests.push_back(std::make_tuple(testDetails, timeTaken));
+}
+
+void OutputRecord::ReportAllTestsComplete(size_t testCount, size_t skipped, size_t failed, Time::Duration totalTime)
+{
+    summaryCount = testCount;
+    summarySkipped = skipped;
+    summaryFailed = failed;
+    summaryDuration = totalTime;
+}
+
+}}

Tests/UnitTests/Helpers/OutputRecord.h

+#ifndef OUTPUTRECORD_H_
+#define OUTPUTRECORD_H_
+
+#include <memory>
+#include <mutex>
+#include <tuple>
+#include <vector>
+#include "xUnit++/IOutput.h"
+
+namespace xUnitpp { namespace Tests {
+
+class OutputRecord : public IOutput
+{
+public:
+    virtual void ReportStart(const TestDetails &testDetails) override;
+    virtual void ReportFailure(const TestDetails &testDetails, const std::string &msg, const LineInfo &lineInfo) override;
+    virtual void ReportSkip(const TestDetails &testDetails, const std::string &reason) override;
+    virtual void ReportFinish(const TestDetails &testDetails, Time::Duration timeTaken) override;
+    virtual void ReportAllTestsComplete(size_t testCount, size_t skipped, size_t failed, Time::Duration totalTime) override;
+
+    std::vector<TestDetails> orderedTestList;
+    std::vector<std::tuple<TestDetails, std::string, LineInfo>> failures;
+    std::vector<std::tuple<TestDetails, std::string>> skips;
+    std::vector<std::tuple<TestDetails, Time::Duration>> finishedTests;
+
+    size_t summaryCount;
+    size_t summarySkipped;
+    size_t summaryFailed;
+    Time::Duration summaryDuration;
+
+private:
+    std::mutex lock;
+};
+
+}}
+
+#endif

Tests/UnitTests/LineInfo.cpp

 
     xUnitpp::AttributeCollection attributes;
     xUnitpp::TestCollection collection;
-    xUnitpp::Check localCheck;
+    auto localCheck = std::make_shared<xUnitpp::Check>();
     xUnitpp::TestCollection::Register reg(collection, test,
         "LineInfoOverridesDefaultTestLineInfo", "LineInfo", attributes,
         -1, __FILE__, __LINE__, localCheck);

Tests/UnitTests/TestRunner.cpp

+#include <thread>
+#include "xUnit++/xUnit++.h"
+#include "xUnit++/xUnitTestRunner.h"
+#include "xUnit++/xUnitTime.h"
+#include "Helpers/OutputRecord.h"
+
+using xUnitpp::Assert;
+using xUnitpp::xUnitTest;
+using xUnitpp::RunTests;
+namespace Tests = xUnitpp::Tests;
+namespace Time = xUnitpp::Time;
+
+SUITE(TestRunner)
+{
+
+struct TestFactory
+{
+    TestFactory(std::function<void()> testFn, std::shared_ptr<xUnitpp::Check> check)
+        : testFn(testFn)
+        , check(check)
+        , timeLimit(-1)
+        , file("dummy.cpp")
+        , line(0)
+    {
+    }
+
+    TestFactory &Name(const std::string &name)
+    {
+        this->name = name;
+        return *this;
+    }
+
+    TestFactory &Suite(const std::string &suite)
+    {
+        this->suite = suite;
+        return *this;
+    }
+
+    TestFactory &Duration(Time::Duration timeLimit)
+    {
+        this->timeLimit = timeLimit;
+        return *this;
+    }
+
+    TestFactory &Attributes(const xUnitpp::AttributeCollection &attributes)
+    {
+        this->attributes = attributes;
+        return *this;
+    }
+
+    TestFactory &TestFile(const std::string &file)
+    {
+        this->file = file;
+        return *this;
+    }
+
+    TestFactory &TestLine(int line)
+    {
+        this->line = line;
+        return *this;
+    }
+
+    operator xUnitTest() const
+    {
+        return xUnitTest(testFn, name, suite, attributes, timeLimit, file, line, check);
+    }
+
+private:
+    std::function<void()> testFn;
+    std::shared_ptr<xUnitpp::Check> check;
+    std::string name;
+    std::string suite;
+    Time::Duration timeLimit;
+    xUnitpp::AttributeCollection attributes;
+    std::string file;
+    int line;
+};
+
+struct TestRunnerFixture
+{
+    TestRunnerFixture()
+        : testCheck(std::make_shared<xUnitpp::Check>())
+    {
+    }
+
+    std::shared_ptr<xUnitpp::Check> testCheck;
+    std::vector<xUnitTest> tests;
+    Tests::OutputRecord output;
+    Time::Duration duration;
+};
+
+namespace Filter
+{
+    bool AllTests(const xUnitpp::TestDetails &) { return true; }
+    bool NoTests(const xUnitpp::TestDetails &) { return false; }
+}
+
+struct EmptyTest
+{
+    void operator()() const
+    {
+    }
+};
+
+struct FailingTest
+{
+    void operator()() const
+    {
+        Assert.Fail();
+    }
+};
+
+struct SleepyTest
+{
+    SleepyTest(int ms = 20)
+        : duration(ms)
+    {
+    }
+
+    void operator()() const
+    {
+        std::this_thread::sleep_for(duration);
+    }
+
+    std::chrono::milliseconds duration;
+};
+
+FACT_FIXTURE(TestStartIsReported, TestRunnerFixture)
+{
+    tests.push_back(TestFactory(EmptyTest(), testCheck).Name("started"));
+    RunTests(output, &Filter::AllTests, tests, duration, 0);
+
+    Assert.Equal(1U, output.orderedTestList.size());
+    Assert.Equal("started", output.orderedTestList[0].Name);
+}
+
+FACT_FIXTURE(TestFinishIsReported, TestRunnerFixture)
+{
+    tests.push_back(TestFactory(EmptyTest(), testCheck).Name("finished"));
+    RunTests(output, &Filter::AllTests, tests, duration, 0);
+
+    Assert.Equal(1U, output.finishedTests.size());
+    Assert.Equal("finished", std::get<0>(output.finishedTests[0]).Name);
+}
+
+FACT_FIXTURE(TestFinishIncludesCorrectTime, TestRunnerFixture)
+{
+    auto test = SleepyTest();
+
+    tests.push_back(TestFactory(test, testCheck));
+    RunTests(output, &Filter::AllTests, tests, duration, 0);
+
+    auto min = Time::ToDuration(test.duration) - Time::ToDuration(Time::ToMilliseconds(5));
+    auto max = Time::ToDuration(test.duration) + Time::ToDuration(Time::ToMilliseconds(5));
+
+    Assert.InRange(std::get<1>(output.finishedTests[0]).count(), min.count(), max.count());
+}
+
+FACT_FIXTURE(NoTestsAreFailuresWhenNoTestsRun, TestRunnerFixture)
+{
+    tests.push_back(TestFactory(EmptyTest(), testCheck).Name("not run"));
+
+    Assert.Equal(0, RunTests(output, &Filter::NoTests, tests, duration, 0));
+    Assert.Equal(0U, output.orderedTestList.size());
+    Assert.Equal(0U, output.finishedTests.size());
+}
+
+FACT_FIXTURE(FailureIsReportedOncePerAssert, TestRunnerFixture)
+{
+    tests.push_back(TestFactory(FailingTest(), testCheck).Name("failing"));
+    tests.push_back(TestFactory(EmptyTest(), testCheck).Name("empty"));
+    tests.push_back(TestFactory(FailingTest(), testCheck).Name("failing"));
+
+    Assert.Equal(2, RunTests(output, &Filter::AllTests, tests, duration, 0));
+    Assert.Equal(2U, output.failures.size());
+    
+}
+
+FACT_FIXTURE(TestsAbortOnFirstAssert, TestRunnerFixture)
+{
+    tests.push_back(TestFactory([]() { Assert.Fail() << "first"; Assert.Fail() << "second"; }, testCheck));
+
+    Assert.Equal(1, RunTests(output, &Filter::AllTests, tests, duration, 0), LI);
+    Assert.Equal(1U, output.failures.size(), LI);
+    Assert.Contains(std::get<1>(output.failures[0]), "first", LI);
+    Assert.DoesNotContain(std::get<1>(output.failures[0]), "second", LI);
+}
+
+FACT_FIXTURE(FailureIsReportedOncePerCheck, TestRunnerFixture)
+{
+    tests.push_back(TestFactory([=]() { testCheck->Fail(); }, testCheck));
+
+    Assert.Equal(1, RunTests(output, &Filter::AllTests, tests, duration, 0));
+    Assert.Equal(1U, output.failures.size());
+}
+
+FACT_FIXTURE(TestsDoNotAbortOnCheck, TestRunnerFixture)
+{
+    tests.push_back(TestFactory([=]() { testCheck->Fail() << "first"; testCheck->Fail() << "second"; }, testCheck));
+
+    Assert.Equal(1, RunTests(output, &Filter::AllTests, tests, duration, 0));
+    Assert.Equal(2U, output.failures.size());
+    Assert.Contains(std::get<1>(output.failures[0]), "first");
+    Assert.Contains(std::get<1>(output.failures[1]), "second");
+}
+
+FACT_FIXTURE(TestCountIsReported, TestRunnerFixture)
+{
+    tests.push_back(TestFactory(EmptyTest(), testCheck));
+    RunTests(output, &Filter::AllTests, tests, duration, 0);
+
+    Assert.Equal(1U, output.summaryCount);
+}
+
+FACT_FIXTURE(FailedTestsAreReported, TestRunnerFixture)
+{
+    tests.push_back(TestFactory(FailingTest(), testCheck));
+    RunTests(output, &Filter::AllTests, tests, duration, 0);
+
+    Assert.Equal(1U, output.summaryFailed);
+}
+
+FACT_FIXTURE(FailuresAreReported, TestRunnerFixture)
+{
+    tests.push_back(TestFactory(FailingTest(), testCheck));
+    tests.push_back(TestFactory([=]() { testCheck->Fail(); testCheck->Fail(); }, testCheck));
+    RunTests(output, &Filter::AllTests, tests, duration, 0);
+
+    Assert.Equal(3U, output.failures.size());
+}
+
+FACT_FIXTURE(SkippedTestsAreReported, TestRunnerFixture)
+{
+    xUnitpp::AttributeCollection attributes;
+    attributes.insert(std::make_pair("Skip", ""));
+
+    tests.push_back(TestFactory(FailingTest(), testCheck).Attributes(attributes));
+
+    RunTests(output, &Filter::AllTests, tests, duration, 0);
+
+    Assert.Equal(1U, output.summarySkipped);
+}
+
+TIMED_FACT_FIXTURE(SlowTestsPassHighTimeThreshold, TestRunnerFixture, 0)
+{
+    tests.push_back(TestFactory(SleepyTest(), testCheck));
+    RunTests(output, &Filter::AllTests, tests, Time::ToDuration(Time::ToMilliseconds(200)), 0);
+
+    Assert.Equal(0U, output.failures.size());
+    Assert.Equal(0U, output.summaryFailed);
+}
+
+TIMED_FACT_FIXTURE(SlowTestsFailLowTimeThreshold, TestRunnerFixture, 0)
+{
+    SleepyTest sleepyTest;
+    tests.push_back(TestFactory(sleepyTest, testCheck));
+    RunTests(output, &Filter::AllTests, tests, Time::ToDuration(Time::ToMilliseconds(1)), 0);
+
+    Assert.Equal(1U, output.failures.size());
+    Assert.Equal(1U, output.summaryFailed);
+}
+
+TIMED_FACT_FIXTURE(SlowTestFailsBecauseOfTimeLimitReportsReason, TestRunnerFixture, 0)
+{
+    tests.push_back(TestFactory(SleepyTest(), testCheck));
+    RunTests(output, &Filter::AllTests, tests, Time::ToDuration(Time::ToMilliseconds(1)), 0);
+
+    Assert.Equal(1U, output.failures.size());
+    Assert.Contains(std::get<1>(output.failures[0]), "Test failed to complete within");
+    Assert.Contains(std::get<1>(output.failures[0]), "1 milliseconds.");
+}
+
+TIMED_FACT_FIXTURE(SlowTestWithTimeExemptionPasses, TestRunnerFixture, 0)
+{
+    tests.push_back(TestFactory(SleepyTest(), testCheck).Duration(Time::ToDuration(Time::ToMilliseconds(0))));
+
+    Assert.Equal(0, RunTests(output, &Filter::AllTests, tests, Time::ToDuration(Time::ToMilliseconds(1)), 0));
+}
+
+FACT_FIXTURE(AllTestsAreRunWithNoFilter, TestRunnerFixture)
+{
+    tests.push_back(TestFactory(EmptyTest(), testCheck));
+    RunTests(output, &Filter::AllTests, tests, duration, 0);
+
+    Assert.Equal(1U, output.summaryCount);
+}
+
+FACT_FIXTURE(FilteredTestsDoNotRun, TestRunnerFixture)
+{
+    tests.push_back(TestFactory(EmptyTest(), testCheck));
+    RunTests(output, &Filter::NoTests, tests, duration, 0);
+
+    Assert.Equal(0U, output.summaryCount);
+}
+
+ATTRIBUTES(TestOrderIsRandomized, ("Skip", "Threading issues preventing me from tracking down why this *doesn't* fail right now."))
+TIMED_FACT_FIXTURE(TestOrderIsRandomized, TestRunnerFixture, 0)
+{
+    // the best I can do here is have 10 tests run, and hope that I don't run into the same 10
+    // values back-to-back
+    int originalOrder[10];
+    for (int i = 0; i != 10; ++i)
+    {
+        originalOrder[i] = i;
+        tests.push_back(TestFactory(EmptyTest(), testCheck).TestLine(i));
+    }
+
+    RunTests(output, &Filter::AllTests, tests, duration, 1);
+
+    Assert.NotEqual(std::begin(originalOrder), std::end(originalOrder), output.orderedTestList.begin(), output.orderedTestList.end(),
+        [](int a, const xUnitpp::TestDetails &b)
+        {
+            return a == b.Line;
+        });
+}
+
+}

Tests/UnitTests/Theory.cpp

 
 public:
     TheoryFixture()
-        : localRunner(emptyReporter)
+        : localCheck(std::make_shared<xUnitpp::Check>())
     {
     }
 
 
     void Run()
     {
-        localRunner.RunTests([](const xUnitpp::TestDetails &) { return true; },
+        RunTests(emptyReporter, [](const xUnitpp::TestDetails &) { return true; },
             collection.Tests(), xUnitpp::Time::Duration::zero(), 0);
     }
 
 
     xUnitpp::AttributeCollection attributes;
     xUnitpp::TestCollection collection;
-    xUnitpp::TestRunner localRunner;
-    xUnitpp::Check localCheck;
+    std::shared_ptr<xUnitpp::Check> localCheck;
 };
 
 std::vector<std::tuple<int>> RawFunctionProvider()

Tests/UnitTests/UnitTests.vcxproj

     <ClCompile Include="Assert.Throws.cpp" />
     <ClCompile Include="Assert.True.cpp" />
     <ClCompile Include="Attributes.cpp" />
+    <ClCompile Include="Helpers\OutputRecord.cpp" />
     <ClCompile Include="LineInfo.cpp" />
+    <ClCompile Include="TestRunner.cpp" />
     <ClCompile Include="Theory.cpp" />
   </ItemGroup>
   <ItemGroup>
       <Project>{25df3961-f288-4a96-ae6b-a4950a00ab8e}</Project>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="Helpers\OutputRecord.h" />
+  </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>

Tests/UnitTests/UnitTests.vcxproj.filters

     <ClCompile Include="Attributes.cpp" />
     <ClCompile Include="Theory.cpp" />
     <ClCompile Include="LineInfo.cpp" />
+    <ClCompile Include="TestRunner.cpp" />
+    <ClCompile Include="Helpers\OutputRecord.cpp">
+      <Filter>Test Helpers</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <Filter Include="Test Helpers">
+      <UniqueIdentifier>{005e2285-5a79-4c3b-9d4d-8ba638459f79}</UniqueIdentifier>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="Helpers\OutputRecord.h">
+      <Filter>Test Helpers</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>
     - ensure tests are actually ran
         - will require new project
     - ensure output formats
-    - ensure all failing assert types actually increment count of failed tests
-    - include new Check asserts
     - several tests (look at Theory.cpp) do not actually assert anything happened
 check Release builds for warnings (and failing tests)
 can runner be made configuration agnostic? (ie, can debug runner load release tests, and vice versa? (and what about x64?)

xUnit++.console/main.cpp

         }
     }
 
-    return forcedFailure ? 1 : totalFailures;
+    return forcedFailure ? 1 : -totalFailures;
 }

xUnit++/src/TestCollection.cpp

 
     extern "C" __declspec(dllexport) int FilteredTestsRunner(int timeLimit, int threadLimit, xUnitpp::IOutput &testReporter, std::function<bool(const xUnitpp::TestDetails &)> filter)
     {
-        return xUnitpp::TestRunner(testReporter).RunTests(filter,
-            xUnitpp::TestCollection::Instance().Tests(), xUnitpp::Time::ToDuration(std::chrono::milliseconds(timeLimit)), threadLimit);
+        return xUnitpp::RunTests(testReporter, filter, xUnitpp::TestCollection::Instance().Tests(),
+            xUnitpp::Time::ToDuration(xUnitpp::Time::ToMilliseconds(timeLimit)), threadLimit);
     }
 }
 
 }
 
 TestCollection::Register::Register(TestCollection &collection, const std::function<void()> &fn, const std::string &name, const std::string &suite,
-                                   const AttributeCollection &attributes, int milliseconds, const std::string &filename, int line, const Check &check)
+                                   const AttributeCollection &attributes, int milliseconds, const std::string &filename, int line, std::shared_ptr<Check> check)
 {
-    collection.mTests.emplace_back(xUnitTest(fn, name, suite, attributes, Time::ToDuration(std::chrono::milliseconds(milliseconds)), filename, line, check));
+    collection.mTests.emplace_back(xUnitTest(fn, name, suite, attributes, Time::ToDuration(Time::ToMilliseconds(milliseconds)), filename, line, check));
 }
 
 const std::vector<xUnitTest> &TestCollection::Tests()

xUnit++/src/xUnitTest.cpp

 
 xUnitTest::xUnitTest(std::function<void()> test, const std::string &name, const std::string &suite,
                      const AttributeCollection &attributes, Time::Duration timeLimit,
-                     const std::string &filename, int line, const Check &check)
+                     const std::string &filename, int line, std::shared_ptr<Check> check)
     : mTest(test)
     , mTestDetails(name, suite, attributes, timeLimit, filename, line)
-    , mCheck(std::cref(check))
+    , mCheck(check)
 {
 }
 
 }
 
 xUnitTest::xUnitTest(xUnitTest &&other)
-    : mCheck(other.mCheck)
 {
     swap(*this, other);
 }
 
 const std::vector<xUnitAssert> &xUnitTest::NonFatalFailures() const
 {
-    return mCheck.get().Failures();
+    return mCheck->Failures();
 }
 
 }

xUnit++/src/xUnitTestRunner.cpp

 #include "xUnitAssert.h"
 #include "xUnitTime.h"
 
+#include <iostream>
+
+namespace
+{
+
+class SharedOutput : public xUnitpp::IOutput
+{
+public:
+    SharedOutput(xUnitpp::IOutput &testReporter)
+        : mOutput(testReporter)
+    {
+    }
+
+    virtual void ReportStart(const xUnitpp::TestDetails &details) override
+    {
+        std::lock_guard<std::mutex> guard(mLock);
+        mOutput.get().ReportStart(details);
+    }
+
+    virtual void ReportFailure(const xUnitpp::TestDetails &details, const std::string &message, const xUnitpp::LineInfo &lineInfo) override 
+    {
+        std::lock_guard<std::mutex> guard(mLock);
+        mOutput.get().ReportFailure(details, message, lineInfo);
+    }
+
+    virtual void ReportSkip(const xUnitpp::TestDetails &details, const std::string &reason) override
+    {
+        std::lock_guard<std::mutex> guard(mLock);
+        mOutput.get().ReportSkip(details, reason);
+    }
+
+    virtual void ReportFinish(const xUnitpp::TestDetails &details, xUnitpp::Time::Duration time) override
+    {
+        std::lock_guard<std::mutex> guard(mLock);
+        mOutput.get().ReportFinish(details, time);
+    }
+
+    virtual void ReportAllTestsComplete(size_t total, size_t skipped, size_t failed, xUnitpp::Time::Duration totalTime) override
+    {
+        mOutput.get().ReportAllTestsComplete(total, skipped, failed, totalTime);
+    }
+
+private:
+    SharedOutput(const SharedOutput &);
+    SharedOutput &operator =(SharedOutput);
+
+private:
+    std::mutex mLock;
+    std::reference_wrapper<xUnitpp::IOutput> mOutput;
+};
+
+class AttachedOutput : public xUnitpp::IOutput
+{
+public:
+    AttachedOutput(SharedOutput &output)
+        : mAttached(true)
+        , mOutput(std::ref(output))
+    {
+    }
+
+    void Detach()
+    {
+        std::lock_guard<std::mutex> guard(mLock);
+        mAttached = false;
+    }
+
+    virtual void ReportStart(const xUnitpp::TestDetails &details) override
+    {
+        std::lock_guard<std::mutex> guard(mLock);
+
+        if (mAttached)
+        {
+            mOutput.get().ReportStart(details);
+        }
+    }
+
+    virtual void ReportFailure(const xUnitpp::TestDetails &details, const std::string &message, const xUnitpp::LineInfo &lineInfo) override 
+    {
+        std::lock_guard<std::mutex> guard(mLock);
+
+        if (mAttached)
+        {
+            mOutput.get().ReportFailure(details, message, lineInfo);
+        }
+    }
+
+    virtual void ReportSkip(const xUnitpp::TestDetails &details, const std::string &reason) override
+    {
+        std::lock_guard<std::mutex> guard(mLock);
+
+        if (mAttached)
+        {
+            mOutput.get().ReportSkip(details, reason);
+        }
+    }
+
+    virtual void ReportFinish(const xUnitpp::TestDetails &details, xUnitpp::Time::Duration time) override
+    {
+        std::lock_guard<std::mutex> guard(mLock);
+
+        if (mAttached)
+        {
+            mOutput.get().ReportFinish(details, time);
+        }
+    }
+
+    virtual void ReportAllTestsComplete(size_t, size_t, size_t, xUnitpp::Time::Duration) override
+    {
+        throw std::logic_error("No one holding an AttachedOutput object should be calling ReportAllTestsComplete.");
+    }
+
+private:
+    AttachedOutput(const AttachedOutput &);
+    AttachedOutput &operator =(AttachedOutput);
+
+private:
+    std::mutex mLock;
+    bool mAttached;
+    std::reference_wrapper<SharedOutput> mOutput;
+};
+
+}
+
 namespace xUnitpp
 {
 
-class TestRunner::Impl
-{
-public:
-    Impl(IOutput &testReporter)
-        : mTestReporter(testReporter)
-    {
-    }
-
-    void OnTestStart(const TestDetails &details)
-    {
-        std::lock_guard<std::mutex> guard(mStartMtx);
-        mTestReporter.ReportStart(details);
-    }
-
-    void OnTestFailure(const TestDetails &details, const std::string &message, const LineInfo &lineInfo)
-    {
-        std::lock_guard<std::mutex> guard(mFailureMtx);
-        mTestReporter.ReportFailure(details, message, lineInfo);
-    }
-
-    void OnTestSkip(const TestDetails &details, const std::string &reason)
-    {
-        mTestReporter.ReportSkip(details, reason);
-    }
-
-    void OnTestFinish(const TestDetails &details, Time::Duration time)
-    {
-        std::lock_guard<std::mutex> guard(mFinishMtx);
-        mTestReporter.ReportFinish(details, time);
-    }
-
-
-    void OnAllTestsComplete(int total, int skipped, int failed, Time::Duration totalTime)
-    {
-        mTestReporter.ReportAllTestsComplete(total, skipped, failed, totalTime);
-    }
-
-private:
-    Impl(const Impl &);
-    Impl &operator=(Impl);
-
-private:
-    IOutput &mTestReporter;
-
-    std::mutex mStartMtx;
-    std::mutex mFailureMtx;
-    std::mutex mFinishMtx;
-};
-
-TestRunner::TestRunner(IOutput &testReporter)
-    : mImpl(new Impl(testReporter))
-{
-}
-
-int TestRunner::RunTests(std::function<bool(const TestDetails &)> filter, const std::vector<xUnitTest> &tests, Time::Duration maxTestRunTime, size_t maxConcurrent)
+int RunTests(IOutput &output, std::function<bool(const TestDetails &)> filter, const std::vector<xUnitTest> &tests, Time::Duration maxTestRunTime, size_t maxConcurrent)
 {
     auto timeStart = std::chrono::system_clock::now();
 
     std::atomic<int> failedTests = 0;
     int skippedTests = 0;
 
+    SharedOutput sharedOutput(output);
+
     std::vector<xUnitTest> activeTests;
     std::copy_if(tests.begin(), tests.end(), std::back_inserter(activeTests), [&filter](const xUnitTest &test) { return filter(test.TestDetails()); });
 
+    // leaving commented out until I can figure out why the test doesn't fail
+    //std::random_shuffle(activeTests.begin(), activeTests.end());
+
     std::vector<std::future<void>> futures;
     for (auto &test : activeTests)
     {
             if (skip != test.TestDetails().Attributes.end())
             {
                 skippedTests++;
-                mImpl->OnTestSkip(test.TestDetails(), skip->second);
+                sharedOutput.ReportSkip(test.TestDetails(), skip->second);
                 continue;
             }
         }
                     ThreadCounter &tc;
                 } counterGuard(threadCounter);
 
-                auto actualTest = [&](bool reportEnd) -> Time::TimeStamp
+                //
+                // We are deliberately not capturing any values by reference, since the thread running this lambda may be detached
+                // and abandoned by a timed test. If that were to happen, variables on the stack would get destroyed out from underneath us.
+                // Instead, we're going to make copies that are guaranteed to outlive our method, and return the test status.
+                // If the running thread is still valid, it can manage updating the count of failed threads if necessary.
+                auto actualTest = [](bool reportEnd, xUnitTest runningTest, std::shared_ptr<AttachedOutput> output)-> std::tuple<Time::TimeStamp, bool>
                     {
                         bool failed = false;
                         Time::TimeStamp testStart;
 
                         auto CheckNonFatalErrors = [&]()
                         {
-                            if (!failed && !test.NonFatalFailures().empty())
+                            if (!failed && !runningTest.NonFatalFailures().empty())
                             {
                                 failed = true;
-                                for (const auto &assert : test.NonFatalFailures())
+                                for (auto &assert : runningTest.NonFatalFailures())
                                 {
-                                    mImpl->OnTestFailure(test.TestDetails(), assert.what(), assert.LineInfo());
+                                    output->ReportFailure(runningTest.TestDetails(), assert.what(), assert.LineInfo());
                                 }
                             }
                         };
                         
                         try
                         {
-                            mImpl->OnTestStart(test.TestDetails());
+                            output->ReportStart(runningTest.TestDetails());
 
                             testStart = Time::Clock::now();
-                            test.Run();
+                            runningTest.Run();
                         }
                         catch (const xUnitAssert &e)
                         {
                             CheckNonFatalErrors();
-                            mImpl->OnTestFailure(test.TestDetails(), e.what(), e.LineInfo());
+                            output->ReportFailure(runningTest.TestDetails(), e.what(), e.LineInfo());
                             failed = true;
                         }
                         catch (const std::exception &e)
                         {
                             CheckNonFatalErrors();
-                            mImpl->OnTestFailure(test.TestDetails(), e.what(), LineInfo::empty());
+                            output->ReportFailure(runningTest.TestDetails(), e.what(), LineInfo::empty());
                             failed = true;
                         }
                         catch (...)
                         {
                             CheckNonFatalErrors();
-                            mImpl->OnTestFailure(test.TestDetails(), "Unknown exception caught: test has crashed", LineInfo::empty());
+                            output->ReportFailure(runningTest.TestDetails(), "Unknown exception caught: test has crashed", LineInfo::empty());
                             failed = true;
                         }
 
                         CheckNonFatalErrors();
 
-                        if (failed)
+                        if (reportEnd)
                         {
-                            ++failedTests;
+                            output->ReportFinish(runningTest.TestDetails(), Time::ToDuration(Time::Clock::now() - testStart));
                         }
 
-                        if (reportEnd)
-                        {
-                            mImpl->OnTestFinish(test.TestDetails(), Time::ToDuration(Time::Clock::now() - testStart));
-                        }
-
-                        return testStart;
+                        return std::make_tuple(testStart, failed);
                     };
 
                 auto testTimeLimit = test.TestDetails().TimeLimit;
                     // note that forcing a test to run in under a certain amount of time is inherently fragile
                     // there's no guarantee that a thread, once started, actually gets `maxTestRunTime` nanoseconds of CPU
 
-                    Time::TimeStamp testStart;
+                    auto m = std::make_shared<std::mutex>();
+                    std::unique_lock<std::mutex> gate(*m);
 
-                    std::mutex m;
-                    std::unique_lock<std::mutex> gate(m);
+                    auto attachedOutput = std::make_shared<AttachedOutput>(sharedOutput);
+                    auto threadStarted = std::make_shared<std::condition_variable>();
+                    auto testStart = std::make_shared<Time::TimeStamp>();
+                    auto failed = std::make_shared<bool>();
+                    std::thread timedRunner([=]()
+                        {
+                            m->lock();
+                            m->unlock();
 
-                    auto threadStarted = std::make_shared<std::condition_variable>();
-                    std::thread timedRunner([&, threadStarted]()
-                        {
-                            m.lock();
-                            m.unlock();
+                            auto result = actualTest(false, test, attachedOutput);
+                            *testStart = std::get<0>(result);
+                            *failed = std::get<1>(result);
 
-                            testStart = actualTest(false);
                             threadStarted->notify_all();
                         });
                     timedRunner.detach();
 
                     if (threadStarted->wait_for(gate, std::chrono::duration_cast<std::chrono::nanoseconds>(testTimeLimit)) == std::cv_status::timeout)
                     {
-                        mImpl->OnTestFailure(test.TestDetails(), "Test failed to complete within " + std::to_string(Time::ToMilliseconds(testTimeLimit).count()) + " milliseconds.", LineInfo::empty());
-                        mImpl->OnTestFinish(test.TestDetails(), testTimeLimit);
+                        attachedOutput->Detach();
+                        sharedOutput.ReportFailure(test.TestDetails(), "Test failed to complete within " + std::to_string(Time::ToMilliseconds(testTimeLimit).count()) + " milliseconds.", LineInfo::empty());
+                        sharedOutput.ReportFinish(test.TestDetails(), testTimeLimit);
                         ++failedTests;
                     }
                     else
                     {
-                        mImpl->OnTestFinish(test.TestDetails(), Time::ToDuration(Time::Clock::now() - testStart));
+                        if (*failed)
+                        {
+                            ++failedTests;
+                        }
+
+                        sharedOutput.ReportFinish(test.TestDetails(), Time::ToDuration(Time::Clock::now() - *testStart));
                     }
                 }
                 else
                 {
-                    actualTest(true);
+                    auto result = actualTest(true, test, std::make_shared<AttachedOutput>(sharedOutput));
+
+                    if (std::get<1>(result))
+                    {
+                        ++failedTests;
+                    }
                 }
             }));
     }
         test.get();
     }
     
-    mImpl->OnAllTestsComplete((int)futures.size(), skippedTests, failedTests, Time::ToDuration(Time::Clock::now() - timeStart));
+    sharedOutput.ReportAllTestsComplete((int)futures.size(), skippedTests, failedTests, Time::ToDuration(Time::Clock::now() - timeStart));
 
-    return -failedTests;
+    return failedTests;
 }
 
 }

xUnit++/xUnit++/TestCollection.h

 
     public:
         Register(TestCollection &collection, const std::function<void()> &fn, const std::string &name, const std::string &suite,
-            const AttributeCollection &attributes, int milliseconds, const std::string &filename, int line, const Check &check);
+            const AttributeCollection &attributes, int milliseconds, const std::string &filename, int line, std::shared_ptr<Check> check);
 
         template<typename TTheory, typename TTheoryData>
         Register(TestCollection &collection, TTheory theory, TTheoryData theoryData, const std::string &name, const std::string &suite,
-            const AttributeCollection &attributes, int milliseconds, const std::string &filename, int line, const Check &check)
+            const AttributeCollection &attributes, int milliseconds, const std::string &filename, int line, std::shared_ptr<Check> check)
         {
             int id = 1;
             for (auto t : theoryData())
                 auto theoryName = name + "(" + std::to_string(id++) + ")";
 
                 collection.mTests.emplace_back(xUnitTest(TheoryHelper(theory, std::move(t)), theoryName, suite,
-                        attributes, Time::ToDuration(std::chrono::milliseconds(milliseconds)), filename, line, check));
+                    attributes, Time::ToDuration(Time::ToMilliseconds(milliseconds)), filename, line, check));
             }
         }
     };

xUnit++/xUnit++/xUnitCheck.h

     Check();
 
 private:
+    Check(const Check &) /* = delete */;
+    Check(Check &&) /* = delete */;
+    Check &operator =(Check) /* = delete */;
+
     const std::vector<xUnitAssert> &Failures() const;
 
 private:

xUnit++/xUnit++/xUnitMacros.h

 #ifndef XUNITMACROS_H_
 #define XUNITMACROS_H_
 
+#include <memory>
 #include <tuple>
 #include <vector>
 #include "Attributes.h"
 #define TIMED_FACT_FIXTURE(FactName, FixtureType, timeout) \
     namespace FactName ## _ns { \
         using xUnitpp::Assert; \
-        xUnitpp::Check Check; \
+        std::shared_ptr<xUnitpp::Check> pCheck = std::make_shared<xUnitpp::Check>(); \
         class FactName ## _Fixture : public FixtureType \
         { \
+            /* !!!VS fix when '= delete' is supported */ \
+            FactName ## _Fixture &operator =(FactName ## _Fixture) /* = delete */; \
         public: \
+            FactName ## _Fixture() : Check(*pCheck) { } \
             void FactName(); \
+            const xUnitpp::Check &Check; \
         }; \
         void FactName ## _runner() { FactName ## _Fixture().FactName(); } \
         xUnitpp::TestCollection::Register reg(xUnitpp::TestCollection::Instance(), \
             &FactName ## _runner, #FactName, xUnitSuite::Name(), \
-            xUnitAttributes::Attributes(), timeout, __FILE__, __LINE__, Check); \
+            xUnitAttributes::Attributes(), timeout, __FILE__, __LINE__, pCheck); \
     } \
     void FactName ## _ns::FactName ## _Fixture::FactName()
 
 #define TIMED_DATA_THEORY(TheoryName, params, DataProvider, timeout) \
     namespace TheoryName ## _ns { \
         using xUnitpp::Assert; \
-        xUnitpp::Check Check; \
+        std::shared_ptr<xUnitpp::Check> pCheck = std::make_shared<xUnitpp::Check>(); \
+        xUnitpp::Check &Check = *pCheck; \
         void TheoryName params; \
         xUnitpp::TestCollection::Register reg(xUnitpp::TestCollection::Instance(), \
             TheoryName, DataProvider, #TheoryName, xUnitSuite::Name(), \
-            xUnitAttributes::Attributes(), timeout, __FILE__, __LINE__, Check); \
+            xUnitAttributes::Attributes(), timeout, __FILE__, __LINE__, pCheck); \
     } \
     void TheoryName ## _ns::TheoryName params
 
 #define TIMED_THEORY(TheoryName, params, timeout, ...) \
     namespace TheoryName ## _ns { \
         using xUnitpp::Assert; \
-        xUnitpp::Check Check; \
+        std::shared_ptr<xUnitpp::Check> pCheck = std::make_shared<xUnitpp::Check>(); \
+        xUnitpp::Check &Check = *pCheck; \
         void TheoryName params; \
         decltype(FIRST_ARG(__VA_ARGS__)) args[] = { __VA_ARGS__ }; \
         xUnitpp::TestCollection::Register reg(xUnitpp::TestCollection::Instance(), \
             TheoryName, xUnitpp::TheoryData(PP_NARGS(__VA_ARGS__), args), #TheoryName, \
-            xUnitSuite::Name(), xUnitAttributes::Attributes(), timeout, __FILE__, __LINE__, Check); \
+            xUnitSuite::Name(), xUnitAttributes::Attributes(), timeout, __FILE__, __LINE__, pCheck); \
     } \
     void TheoryName ## _ns::TheoryName params
 

xUnit++/xUnit++/xUnitTest.h

 #define XUNITTEST_H_
 
 #include <functional>
+#include <memory>
 #include <string>
 #include "TestDetails.h"
 #include "xUnitCheck.h"
 public:
     xUnitTest(std::function<void()> test, const std::string &name, const std::string &suite,
         const AttributeCollection &attributes, Time::Duration timeLimit,
-        const std::string &filename, int line, const Check &check);
+        const std::string &filename, int line, std::shared_ptr<Check> check);
     xUnitTest(const xUnitTest &other);
     xUnitTest(xUnitTest &&other);
     xUnitTest &operator =(xUnitTest other);
 private:
     std::function<void()> mTest;
     xUnitpp::TestDetails mTestDetails;
-    std::reference_wrapper<const Check> mCheck;
+    std::shared_ptr<Check> mCheck;
 };
 
 }

xUnit++/xUnit++/xUnitTestRunner.h

 struct TestDetails;
 class xUnitTest;
 
-class TestRunner
-{
-public:
-    TestRunner(IOutput &testReporter);
-    int RunTests(std::function<bool(const TestDetails &)> filter, const std::vector<xUnitTest> &tests,
+int RunTests(IOutput &output, std::function<bool(const TestDetails &)> filter, const std::vector<xUnitTest> &tests,
                  Time::Duration maxTestRunTime, size_t maxConcurrent);
 
-private:
-    class Impl;
-    std::shared_ptr<Impl> mImpl;
-};
-
 }
 
 #endif