Commits

Matt Oswald committed 95b98f5

failing theories include (0-based) index of failing data set in output

Comments (0)

Files changed (4)

xUnit++/DefaultReporter.cpp

 
 namespace DefaultReporter
 {
-    void ReportStart(const TestDetails &)
+    void ReportStart(const TestDetails &, int)
     {
     }
 
-    void ReportFailure(const TestDetails &testDetails, const std::string &msg)
+    void ReportFailure(const TestDetails &testDetails, int dataIndex, const std::string &msg)
     {
-        std::cerr << (testDetails.Filename + "(" + std::to_string(testDetails.Line) +
-            "): error " + testDetails.Name + ": " + msg + "\n");
+        if (dataIndex < 0)
+        {
+            std::cerr << (testDetails.Filename + "(" + std::to_string(testDetails.Line) +
+                "): error " + testDetails.Name + ": " + msg + "\n");
+        }
+        else
+        {
+            std::cerr << (testDetails.Filename + "(" + std::to_string(testDetails.Line) +
+                "): error " + testDetails.Name + "(" + std::to_string(dataIndex) + "): " + msg + "\n");
+        }
     }
 
-    void ReportFinish(const TestDetails &, std::chrono::milliseconds)
+    void ReportFinish(const TestDetails &, int, std::chrono::milliseconds)
     {
     }
 

xUnit++/DefaultReporter.h

 
 namespace DefaultReporter
 {
-    void ReportStart(const TestDetails &);
-    void ReportFailure(const TestDetails &testDetails, const std::string &msg);
-    void ReportFinish(const TestDetails &, std::chrono::milliseconds);
+    void ReportStart(const TestDetails &, int);
+    void ReportFailure(const TestDetails &testDetails, int dataIndex, const std::string &msg);
+    void ReportFinish(const TestDetails &, int, std::chrono::milliseconds);
     void ReportAllTestsComplete(size_t testCount, size_t failureCount, size_t skipped, std::chrono::milliseconds totalTime); 
 }
 

xUnit++/TestRunner.cpp

         TestInstance(const xUnitpp::TestDetails &testDetails, int id, int groupId, int groupSize, std::function<void()> test)
             : testDetails(testDetails)
             , id(id)
+            , dataIndex(-1)
+            , groupId(groupId)
+            , groupSize(groupSize)
+            , test(test)
+        {
+        }
+
+        TestInstance(const xUnitpp::TestDetails &testDetails, int id, int dataIndex, int groupId, int groupSize, std::function<void()> test)
+            : testDetails(testDetails)
+            , id(id)
+            , dataIndex(dataIndex)
             , groupId(groupId)
             , groupSize(groupSize)
             , test(test)
         TestInstance(const TestInstance &other)
             : testDetails(other.testDetails)
             , id(other.id)
+            , dataIndex(other.dataIndex)
             , groupId(other.groupId)
             , groupSize(other.groupSize)
             , test(other.test)
 
             swap(ti0.testDetails, ti1.testDetails);
             swap(ti0.id, ti1.id);
+            swap(ti0.dataIndex, ti1.dataIndex);
             swap(ti0.groupId, ti1.groupId);
             swap(ti0.groupSize, ti1.groupSize);
             swap(ti0.test, ti1.test);
         xUnitpp::TestDetails testDetails;
 
         size_t id;
+        size_t dataIndex;
         size_t groupId;
         size_t groupSize;
 
             if (suite == "" || theorySet.TestDetails().Suite == suite)
             {
                 ++groupId;
-
+                size_t dataIndex = 0;
                 for (auto &theory : theorySet.Theories())
                 {
-                    mTests.emplace_back(TestInstance(theorySet.TestDetails(), ++id, groupId, theorySet.Theories().size(), theory));
+                    mTests.emplace_back(TestInstance(theorySet.TestDetails(), ++id, dataIndex++, groupId, theorySet.Theories().size(), theory));
                 }
             }
         }
 class TestRunner::Impl
 {
 public:
-    Impl(std::function<void(const TestDetails &)> onTestStart,
-         std::function<void(const TestDetails &, const std::string &)> onTestFailure,
-         std::function<void(const TestDetails &, std::chrono::milliseconds)> onTestFinish,
+    Impl(std::function<void(const TestDetails &, int)> onTestStart,
+         std::function<void(const TestDetails &, int, const std::string &)> onTestFailure,
+         std::function<void(const TestDetails &, int, std::chrono::milliseconds)> onTestFinish,
          std::function<void(int, int, int, std::chrono::milliseconds)> onAllTestsComplete)
         : mOnTestStart(onTestStart)
         , mOnTestFailure(onTestFailure)
     {
     }
 
-    void OnTestStart(const TestDetails &details)
+    void OnTestStart(const TestDetails &details, int dataIndex)
     {
         std::lock_guard<std::mutex> guard(mStartMtx);
-        mOnTestStart(details);
+        mOnTestStart(details, dataIndex);
     }
 
-    void OnTestFailure(const TestDetails &details, const std::string &message)
+    void OnTestFailure(const TestDetails &details, int dataIndex, const std::string &message)
     {
         std::lock_guard<std::mutex> guard(mFailureMtx);
-        mOnTestFailure(details, message);
+        mOnTestFailure(details, dataIndex, message);
     }
 
-    void OnTestFinish(const TestDetails &details, std::chrono::milliseconds time)
+    void OnTestFinish(const TestDetails &details, int dataIndex, std::chrono::milliseconds time)
     {
         std::lock_guard<std::mutex> guard(mFinishMtx);
-        mOnTestFinish(details, time);
+        mOnTestFinish(details, dataIndex, time);
     }
 
 
     }
 
 private:
-    std::function<void(const TestDetails &)> mOnTestStart;
-    std::function<void(const TestDetails &, const std::string &)> mOnTestFailure;
-    std::function<void(const TestDetails &, std::chrono::milliseconds)> mOnTestFinish;
+    std::function<void(const TestDetails &, int)> mOnTestStart;
+    std::function<void(const TestDetails &, int, const std::string &)> mOnTestFailure;
+    std::function<void(const TestDetails &, int, std::chrono::milliseconds)> mOnTestFinish;
     std::function<void(int, int, int, std::chrono::milliseconds)> mOnAllTestsComplete;
 
     std::mutex mStartMtx;
             .RunTests(TestCollection::Facts(), TestCollection::Theories(), suite, maxTestRunTime, maxConcurrent);
 }
 
-TestRunner::TestRunner(std::function<void(const TestDetails &)> onTestStart,
-                       std::function<void(const TestDetails &, const std::string &)> onTestFailure,
-                       std::function<void(const TestDetails &, std::chrono::milliseconds)> onTestFinish,
+TestRunner::TestRunner(std::function<void(const TestDetails &, int)> onTestStart,
+                       std::function<void(const TestDetails &, int, const std::string &)> onTestFailure,
+                       std::function<void(const TestDetails &, int, std::chrono::milliseconds)> onTestFinish,
                        std::function<void(int, int, int, std::chrono::milliseconds)> onAllTestsComplete)
     : mImpl(new Impl(onTestStart, onTestFailure, onTestFinish, onAllTestsComplete))
 {
                         TimeStamp testStart;
                         try
                         {
-                            mImpl->OnTestStart(test.testDetails);
+                            mImpl->OnTestStart(test.testDetails, test.dataIndex);
 
                             testStart = Clock::now();
                             test.test();
                         }
                         catch (std::exception &e)
                         {
-                            mImpl->OnTestFailure(test.testDetails, e.what());
+                            mImpl->OnTestFailure(test.testDetails, test.dataIndex, e.what());
                             ++failedTests;
                         }
                         catch (...)
                         {
-                            mImpl->OnTestFailure(test.testDetails, "Unknown exception caught: test has crashed");
+                            mImpl->OnTestFailure(test.testDetails, test.dataIndex, "Unknown exception caught: test has crashed");
                             ++failedTests;
                         }
 
                         if (reportEnd)
                         {
-                            mImpl->OnTestFinish(test.testDetails, std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - testStart));
+                            mImpl->OnTestFinish(test.testDetails, test.dataIndex, std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - testStart));
                         }
 
                         return testStart;
 
                     if (threadStarted->wait_for(gate, testTimeLimit) == std::cv_status::timeout)
                     {
-                        mImpl->OnTestFailure(test.testDetails, "Test failed to complete within " + std::to_string(testTimeLimit.count()) + " milliseconds.");
-                        mImpl->OnTestFinish(test.testDetails, testTimeLimit);
+                        mImpl->OnTestFailure(test.testDetails, test.dataIndex, "Test failed to complete within " + std::to_string(testTimeLimit.count()) + " milliseconds.");
+                        mImpl->OnTestFinish(test.testDetails, test.dataIndex, testTimeLimit);
                         ++failedTests;
                     }
                     else
                     {
-                        mImpl->OnTestFinish(test.testDetails, std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - testStart));
+                        mImpl->OnTestFinish(test.testDetails, test.dataIndex, std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - testStart));
                     }
                 }
                 else

xUnit++/TestRunner.h

 class TestRunner
 {
 public:
-    TestRunner(std::function<void(const TestDetails &)> onTestStart,
-               std::function<void(const TestDetails &, const std::string &)> onTestFailure,
-               std::function<void(const TestDetails &, std::chrono::milliseconds)> onTestFinish,
+    TestRunner(std::function<void(const TestDetails &, int)> onTestStart,
+               std::function<void(const TestDetails &, int, const std::string &)> onTestFailure,
+               std::function<void(const TestDetails &, int, std::chrono::milliseconds)> onTestFinish,
                std::function<void(int, int, int, std::chrono::milliseconds)> onAllTestsComplete);
     size_t RunTests(const std::vector<Fact> &facts, const std::vector<Theory> &theories, const std::string &suite,
                     std::chrono::milliseconds maxTestRunTime = std::chrono::milliseconds::zero(), size_t maxConcurrent = 0);