Commits

Matt Oswald  committed 473cd43

test runner now honors test time limits
updated xUnit++ to use std::chrono::milliseconds instead of my own typedef

  • Participants
  • Parent commits f95fd1e

Comments (0)

Files changed (6)

File xUnit++/DefaultReporter.cpp

             " error: Test [" + testDetails.Name + "] failed with: " + msg + "\n");
     }
 
-    void ReportFinish(const TestDetails &, milliseconds)
+    void ReportFinish(const TestDetails &, std::chrono::milliseconds)
     {
     }
 
-    void ReportAllTestsComplete(size_t testCount, size_t failureCount, size_t skipped, milliseconds totalTime)
+    void ReportAllTestsComplete(size_t testCount, size_t failureCount, size_t skipped, std::chrono::milliseconds totalTime)
     {
         std::string total = std::to_string(testCount) + " tests, ";
         std::string failures = std::to_string(failureCount) + " failed, ";

File xUnit++/DefaultReporter.h

 {
     void ReportStart(const TestDetails &);
     void ReportFailure(const TestDetails &testDetails, const std::string &msg);
-    void ReportFinish(const TestDetails &, milliseconds);
-    void ReportAllTestsComplete(size_t testCount, size_t failureCount, size_t skipped, milliseconds totalTime); 
+    void ReportFinish(const TestDetails &, std::chrono::milliseconds);
+    void ReportAllTestsComplete(size_t testCount, size_t failureCount, size_t skipped, std::chrono::milliseconds totalTime); 
 }
 
 }

File xUnit++/TestRunner.cpp

 #include "TestDetails.h"
 #include "xUnitAssert.h"
 
+#include <iostream>
+
 namespace
 {
 
 public:
     Impl(std::function<void(const TestDetails &)> onTestStart,
          std::function<void(const TestDetails &, const std::string &)> onTestFailure,
-         std::function<void(const TestDetails &, milliseconds)> onTestFinish,
-         std::function<void(int, int, int, milliseconds)> onAllTestsComplete)
+         std::function<void(const TestDetails &, std::chrono::milliseconds)> onTestFinish,
+         std::function<void(int, int, int, std::chrono::milliseconds)> onAllTestsComplete)
         : mOnTestStart(onTestStart)
         , mOnTestFailure(onTestFailure)
         , mOnTestFinish(onTestFinish)
         mOnTestFailure(details, message);
     }
 
-    void OnTestFinish(const TestDetails &details, milliseconds time)
+    void OnTestFinish(const TestDetails &details, std::chrono::milliseconds time)
     {
         std::lock_guard<std::mutex> guard(mFinishMtx);
         mOnTestFinish(details, time);
     }
 
 
-    void OnAllTestsComplete(int total, int skipped, int failed, milliseconds totalTime)
+    void OnAllTestsComplete(int total, int skipped, int failed, std::chrono::milliseconds totalTime)
     {
         mOnAllTestsComplete(total, skipped, failed, totalTime);
     }
 private:
     std::function<void(const TestDetails &)> mOnTestStart;
     std::function<void(const TestDetails &, const std::string &)> mOnTestFailure;
-    std::function<void(const TestDetails &, milliseconds)> mOnTestFinish;
-    std::function<void(int, int, int, milliseconds)> mOnAllTestsComplete;
+    std::function<void(const TestDetails &, std::chrono::milliseconds)> mOnTestFinish;
+    std::function<void(int, int, int, std::chrono::milliseconds)> mOnAllTestsComplete;
 
     std::mutex mStartMtx;
     std::mutex mFailureMtx;
 
 TestRunner::TestRunner(std::function<void(const TestDetails &)> onTestStart,
                        std::function<void(const TestDetails &, const std::string &)> onTestFailure,
-                       std::function<void(const TestDetails &, milliseconds)> onTestFinish,
-                       std::function<void(int, int, int, milliseconds)> onAllTestsComplete)
+                       std::function<void(const TestDetails &, std::chrono::milliseconds)> onTestFinish,
+                       std::function<void(int, int, int, std::chrono::milliseconds)> onAllTestsComplete)
     : mImpl(new Impl(onTestStart, onTestFailure, onTestFinish, onAllTestsComplete))
 {
 }
                     ThreadCounter &tc;
                 } counterGuard(threadCounter);
 
-                decltype(Clock::now()) testStart;
-                try
+                auto actualTest = [&](bool reportEnd) -> TimeStamp
+                    {
+                        TimeStamp testStart;
+                        try
+                        {
+                            mImpl->OnTestStart(test.testDetails);
+
+                            testStart = Clock::now();
+                            test.test();
+                        }
+                        catch (std::exception &e)
+                        {
+                            mImpl->OnTestFailure(test.testDetails, e.what());
+                            ++failedTests;
+                        }
+                        catch (...)
+                        {
+                            mImpl->OnTestFailure(test.testDetails, "Unknown exception caught: test has crashed");
+                            ++failedTests;
+                        }
+
+                        if (reportEnd)
+                        {
+                            mImpl->OnTestFinish(test.testDetails, std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - testStart));
+                        }
+
+                        return testStart;
+                    };
+
+                if (maxTestRunTime > 0)
                 {
-                    mImpl->OnTestStart(test.testDetails);
+                    //
+                    // note that forcing a test to run in under a certain amount of time is inherently fragile
+                    // there's no guarantee that a thread, once started, actually gets `maxTestRunTime` milliseconds of CPU
 
-                    testStart = Clock::now();
-                    test.test();
+                    TimeStamp testStart;
+
+                    std::mutex m;
+                    std::unique_lock<std::mutex> gate(m);
+
+                    std::condition_variable threadStarted;
+                    std::thread timedRunner([&]()
+                        {
+                            m.lock();
+                            m.unlock();
+
+                            testStart = actualTest(false);
+                            threadStarted.notify_all();
+                        });
+                    timedRunner.detach();
+
+                    if (threadStarted.wait_for(gate, std::chrono::milliseconds(maxTestRunTime)) == std::cv_status::timeout)
+                    {
+                        mImpl->OnTestFailure(test.testDetails, "Test failed to complete within " + std::to_string(maxTestRunTime) + " milliseconds.");
+                        mImpl->OnTestFinish(test.testDetails, std::chrono::milliseconds(maxTestRunTime));
+                        ++failedTests;
+                    }
+                    else
+                    {
+                        mImpl->OnTestFinish(test.testDetails, std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - testStart));
+                    }
                 }
-                catch (std::exception &e)
+                else
                 {
-                    mImpl->OnTestFailure(test.testDetails, e.what());
-                    ++failedTests;
+                    actualTest(true);
                 }
-                catch (...)
-                {
-                    mImpl->OnTestFailure(test.testDetails, "Unknown exception caught: test has crashed");
-                    ++failedTests;
-                }
-
-                mImpl->OnTestFinish(test.testDetails, Clock::now() - testStart);
             }));
     }
 
         test.get();
     }
     
-    mImpl->OnAllTestsComplete(futures.size(), failedTests, 0, Clock::now() - timeStart);
+    mImpl->OnAllTestsComplete(futures.size(), failedTests, 0, std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - timeStart));
 
     return failedTests;
 }

File xUnit++/TestRunner.h

 public:
     TestRunner(std::function<void(const TestDetails &)> onTestStart,
                std::function<void(const TestDetails &, const std::string &)> onTestFailure,
-               std::function<void(const TestDetails &, milliseconds)> onTestFinish,
-               std::function<void(int, int, int, milliseconds)> onAllTestsComplete);
+               std::function<void(const TestDetails &, std::chrono::milliseconds)> onTestFinish,
+               std::function<void(int, int, int, std::chrono::milliseconds)> onAllTestsComplete);
     size_t RunTests(const std::vector<Fact> &facts, const std::vector<Theory> &theories, const std::string &suite, size_t maxTestRunTime = 0, size_t maxConcurrent = 0);
 
 private:

File xUnit++/main.cpp

 #include <functional>
 #include <iostream>
 #include <string>
+#include <thread>
 #include <tuple>
 #include <vector>
 #include "TestRunner.h"
     }
 }
 
+FACT(LongRunning)
+{
+    std::this_thread::sleep_for(std::chrono::milliseconds(5000));
+}
+
 int main()
 {
-    xUnitpp::RunAllTests();
+    return xUnitpp::RunAllTests("", 50);
 }

File xUnit++/xUnitTime.h

 {
 
 typedef std::chrono::high_resolution_clock Clock;
-typedef std::chrono::duration<float, std::milli> milliseconds;
+typedef decltype(Clock::now()) TimeStamp;
 
 }