Commits

Matt Oswald committed 0e8478f

TestRunner asynchronously runs tests in random order

Comments (0)

Files changed (6)

     return mTestDetails.Suite;
 }
 
-void Fact::Run()
+std::function<void()> Fact::Test() const
 {
-    try
-    {
-        mFact();
-    }
-    catch (xUnitAssert &)
-    {
-    }
-    catch (std::exception &)
-    {
-    }
-    catch (...)
-    {
-    }
+    return mFact;
 }
 
 }
 
     const std::string &Suite() const;
 
-    void Run();
+    std::function<void()> Test() const;
 
 private:
     std::function<void()> mFact;

xUnit++/TestRunner.cpp

 #include <atomic>
 #include <future>
+#include <limits>
+#include <mutex>
+#include <random>
 #include <vector>
 #include "TestRunner.h"
 #include "TestCollection.h"
     return TestRunner().RunTests(TestCollection::Facts(), TestCollection::Theories(), suite, maxTestRunTime, maxConcurrent);
 }
 
-int TestRunner::RunTests(const std::vector<Fact> &allFacts, const std::vector<Theory> &allTheories, const std::string &suite, size_t maxTestRunTime, size_t maxConcurrent)
+TestRunner::TestRunner(/*std::function<void(const TestDetails &)> onTestStart,
+                       std::function<void(const TestDetails &, const std::string &)> onTestFailure,
+                       std::function<void(const TestDetails &, const std::chrono::duration<float, std::milli>)> onTestFinish,
+                       std::function<void(int, int, int, const std::chrono::duration<float, std::milli>)> onAllTestsComplete*/)
+//    : mOnTestStart(onTestStart)
+//    , mOnTestFailure(onTestFailure)
+//    , mOnTestFinish(onTestFinish)
+//    , mOnAllTestsComplete(onAllTestsComplete)
 {
-    std::vector<Fact> facts;
-    std::vector<Theory> theories;
+}
 
-    for (auto &fact : allFacts)
+int TestRunner::RunTests(const std::vector<Fact> &facts, const std::vector<Theory> &theories, const std::string &suite, size_t maxTestRunTime, size_t maxConcurrent)
+{
+    struct TestInstance
+    {
+        TestInstance(int id, int groupId, int groupSize, std::function<void()> test)
+            : id(id)
+            , groupId(groupId)
+            , groupSize(groupSize)
+            , test(test)
+        {
+        }
+
+        size_t id;
+        size_t groupId;
+        size_t groupSize;
+
+        std::function<void()> test;
+    };
+
+    std::vector<TestInstance> allTests;
+    size_t id = 0;
+    size_t groupId = 0;
+
+    for (auto &fact : facts)
     {
         if (suite == "" || fact.Suite() == suite)
         {
-            facts.emplace_back(fact);
+            allTests.emplace_back(TestInstance(++id, ++groupId, 1, fact.Test()));
         }
     }
 
-    for (auto &theory : allTheories)
-    {
-        if (suite == "" || theory.Suite() == suite)
-        {
-            theories.emplace_back(theory);
-        }
-    }
-
-    // !!! figure out how to get fully randomized test ordering, but sequential test output
-
-    auto policy = std::launch::async;
-    std::vector<std::future<void>> futures;
-
-    size_t activeThreads(0);
-    size_t id = 0;
-    
-    for (auto &fact : facts)
-    {
-        futures.push_back(std::async(policy, [&, maxConcurrent, id]()
-            {
-                fact.Run();
-            }));
-
-        ++id;
-    }
-
     for (auto &theorySet : theories)
     {
+        if (suite == "" || theorySet.Suite() == suite)
+        {
+            ++groupId;
+
         for (auto &theory : theorySet.Theories())
         {
-            futures.push_back(std::async(policy, [&, maxConcurrent, id]()
+                allTests.emplace_back(TestInstance(++id, groupId, theorySet.Theories().size(), theory));
+            }
+        }
+    }
+
+    std::shuffle(allTests.begin(), allTests.end(), std::default_random_engine(std::random_device()()));
+
+    if (maxConcurrent == 0)
                 {
-                    theorySet.Run(theory);
+        maxConcurrent = std::numeric_limits<decltype(maxConcurrent)>::max();
+    }
+
+    class ThreadCounter
+    {
+    public:
+        ThreadCounter(size_t maxThreads)
+            : maxThreads(maxThreads)
+            , activeThreads(0)
+        {
+        }
+
+        void operator++()
+        {
+            std::unique_lock<std::mutex> lock(mtx);
+            condition.wait(lock, [&]() { return activeThreads < maxThreads; });
+
+            ++activeThreads;
+        }
+
+        void operator--()
+        {
+            std::lock_guard<std::mutex> guard(mtx);
+            --activeThreads;
+        }
+
+    private:
+        size_t maxThreads;
+        size_t activeThreads;
+        std::mutex mtx;
+        std::condition_variable condition;
+    } threadCounter(maxConcurrent);
+
+    struct CounterGuard
+    {
+        CounterGuard(ThreadCounter &tc)
+            : tc(tc)
+        {
+            ++tc;
+        }
+
+        ~CounterGuard()
+        {
+            --tc;
+        }
+
+    private:
+        ThreadCounter &tc;
+    };
+
+    std::vector<std::future<void>> futures;
+
+    for (auto &test : allTests)
+    {
+        futures.push_back(std::async([&]()
+            {
+                CounterGuard guard(threadCounter);
+
+                try
+                {
+                    test.test();
+                }
+                catch (std::exception &)
+                {
+                }
                 }));
+    }
 
-            ++id;
-        }
+    for (auto &test : futures)
+    {
+        test.get();
     }
 
     return 0;

xUnit++/TestRunner.h

 #ifndef TESTRUNNER_H_
 #define TESTRUNNER_H_
 
+#include <chrono>
 #include <functional>
 #include <string>
 #include <vector>
 class TestRunner
 {
 public:
-    //TestRunner(std::function<void(const TestDetails &testDetail)> onTestStart, 
+    TestRunner(/*std::function<void(const TestDetails &)> onTestStart,
+               std::function<void(const TestDetails &, const std::string &)> onTestFailure,
+               std::function<void(const TestDetails &, const std::chrono::duration<float, std::milli>)> onTestFinish,
+               std::function<void(int, int, int, const std::chrono::duration<float, std::milli>)> onAllTestsComplete*/);
     int RunTests(const std::vector<Fact> &facts, const std::vector<Theory> &theories, const std::string &suite, size_t maxTestRunTime = 0, size_t maxConcurrent = 0);
+
+private:
+    std::function<void(const TestDetails &)> mOnTestStart;
+    std::function<void(const TestDetails &, const std::string &)> mOnTestFailure;
+    std::function<void(const TestDetails &, const std::chrono::duration<float, std::milli>)> mOnTestFinish;
+    std::function<void(int, int, int, const std::chrono::duration<float, std::milli>)> mOnAllTestsComplete;
 };
 
 }

xUnit++/Theory.cpp

     return mTheories;
 }
 
-void Theory::Run(std::function<void()> theory)
-{
-    try
-    {
-        theory();
     }
-    catch (xUnitAssert &)
-    {
-    }
-    catch (std::exception &)
-    {
-    }
-    catch (...)
-    {
-    }
-}
-
-}
 
     const std::string &Suite() const;
 
-    void Run(std::function<void()> theory);
-
     const std::vector<std::function<void()>> &Theories() const;
 
 private:
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.