1. Matt Oswald
  2. xUnit++

Commits

Matt Oswald  committed d331bd9

implemented sorting and grouping of test output (tests are still run in random order)

  • Participants
  • Parent commits fe4ef59
  • Branches default

Comments (0)

Files changed (5)

File xUnit++.console/CommandLine.cpp

View file
         , timeLimit(0)
         , threadLimit(0)
         , shadowCopy(true)
+        , sort(false)
+        , group(false)
     {
     }
 
 
             if (opt[0] == '-')
             {
-                if (opt == "-v")
+                if (opt == "-v" || opt == "--verbose")
                 {
                     options.verbose = true;
                 }
                         return opt + " expects a following test limit count." + Usage(exe());
                     }
                 }
+                else if (opt == "-o" || opt == "--sort")
+                {
+                    options.sort = true;
+                }
+                else if (opt == "-g" || opt == "--group")
+                {
+                    options.sort = true;
+                    options.group = true;
+                }
                 else if (opt == "--no-shadow")
                 {
                     options.shadowCopy = false;
             " <testLibrary>+ [option]+\n"
             "\n"
             "options:\n\n"
-            "  -v                             : Verbose mode: include successful test timing\n"
+            "  -v --verbose                   : Verbose mode: include successful test timing\n"
             "  -l --list                      : Do not run tests, just list the ones that pass the filters\n"
             "  -s --suite <SUITE>+            : Suite(s) of tests to run (substring match)\n"
             "  -n --name <TEST>+              : Test(s) to run (substring match)\n"
             "  -t --timelimit <milliseconds>  : Set the default test time limit\n"
             "  -x --xml <FILENAME>            : Output Xunit-style XML file\n"
             "  -c --concurrent <max tests>    : Set maximum number of concurrent tests\n"
+            "  -o --sort                      : Sort tests by suite and then by test name\n"
+            "  -g --group                     : Group test output under suite headers (implies --sort)\n"
             "     --no-shadow                 : Disable shadow copying the test binaries\n"
             "\n"
             "Tests are selected with an OR operation for inclusive attributes.\n"
             "Tests are excluded with an AND operation for exclusive attributes.\n"
-            "When VALUE is omitted, any attribute with name NAME is matched.\n";
+            "When VALUE is omitted, any attribute with name NAME is matched.\n"
+            "\n"
+            "Sorting and grouping test output causes test results to be cached until after all tests have completed.\n"
+            "Normally, test results are printed as soon as the test is complete.\n";
 
         return "\nusage: " + exe + usage;
     }

File xUnit++.console/CommandLine.h

View file
         int timeLimit;
         int threadLimit;
         bool shadowCopy;
+        bool sort;
+        bool group;
     };
 
     std::string Parse(int argc, char **argv, Options &options);

File xUnit++.console/ConsoleReporter.cpp

View file
             Fragment(Color color, const std::string &message)
                 : color(color)
                 , message(message)
-        {
-        }
+            {
+            }
 
             friend std::ostream &operator <<(std::ostream &stream, const Fragment &fragment)
-        {
+            {
                 return stream << fragment.color << fragment.message;
-        }
+            }
 
             Color color;
             std::string message;
-        }; 
+        };
 
         TestOutput(const xUnitpp::TestDetails &td, bool verbose)
             : testDetails(td)
         {
         }
 
-        ~TestOutput()
+        void Print(bool grouped)
         {
             if (failed || verbose)
             {
+                if (!grouped)
+                {
+                    std::cout << "\n";
+                }
+
                 if (failed)
                 {
-                    std::cout << Fragment(Color::Failure, "\n[ Failure ] ");
+                    std::cout << Fragment(Color::Failure, "[ Failure ] ");
                 }
                 else
                 {
-                    std::cout << Fragment(Color::Success, "\n[ Success ] ");
+                    std::cout << Fragment(Color::Success, "[ Success ] ");
                 }
 
-                if (!testDetails.Suite.empty())
+                if (!grouped)
                 {
-                    std::cout << Fragment(Color::Suite, testDetails.Suite);
-                    std::cout << Fragment(Color::Separator, TestSeparator);
+                    if (!testDetails.Suite.empty())
+                    {
+                        std::cout << Fragment(Color::Suite, testDetails.Suite);
+                        std::cout << Fragment(Color::Separator, TestSeparator);
+                    }
                 }
 
                 std::cout << Fragment(Color::TestName, testDetails.Name + "\n");
                 {
                     std::cout << msg;
                 }
-            } 
+            }
+        }
+
+        void Skip(const std::string &reason)
+        {
+            fragments.emplace_back(Color::FileAndLine, to_string(testDetails.LineInfo));
+            fragments.emplace_back(Color::Separator, ": ");
+            fragments.emplace_back(Color::Skip, reason);
+            fragments.emplace_back(Color::Default, "\n");
         }
 
         TestOutput &operator <<(const xUnitpp::TestEvent &event)
             return *this;
         }
 
+        const xUnitpp::TestDetails &TestDetails() const
+        {
+            return testDetails;
+        }
+
     private:
         TestOutput(const TestOutput &) /* = delete */;
         TestOutput &operator =(TestOutput) /* = delete */;
     };
 
     typedef std::unordered_map<int, std::shared_ptr<TestOutput>> OutputCache;
-    
+
 public:
-    ReportCache(bool verbose)
+    ReportCache(bool verbose, bool sort, bool group)
         : verbose(verbose)
+        , sort(sort)
+        , group(group)
     {
     }
 
 
     TestOutput &Cache(const xUnitpp::TestDetails &td)
     {
-            auto it = cache.find(td.Id);
-            if (it == cache.end())
+        auto it = cache.find(td.Id);
+        if (it == cache.end())
+        {
+            cache.insert(std::make_pair(td.Id, std::make_shared<TestOutput>(td, verbose)));
+        }
+
+        return *cache[td.Id];
+    }
+
+    void Skip(const xUnitpp::TestDetails &testDetails, const std::string &reason)
+    {
+        if (!sort)
+        {
+            Instant(Color::Skip, "\n[ Skipped ] ");
+
+            if (!testDetails.Suite.empty())
             {
-            cache.insert(std::make_pair(td.Id, std::make_shared<TestOutput>(td, verbose)));
+                Instant(Color::Suite, testDetails.Suite);
+                Instant(Color::Separator, TestSeparator);
             }
 
-            return *cache[td.Id];
+            Instant(Color::TestName, testDetails.ShortName + "\n");
+            Instant(Color::FileAndLine, to_string(testDetails.LineInfo) + ": ");
+            Instant(Color::Skip, reason);
+            Instant(Color::Default, "\n");
         }
+        else
+        {
+            Cache(testDetails).Skip(reason);
+        }
+    }
 
     void Finish(const xUnitpp::TestDetails &td)
+    {
+        if (!sort)
         {
             auto it = cache.find(td.Id);
             if (it != cache.end())
             {
+                it->second->Print(false);
                 cache.erase(it);
             }
         }
+    }
 
-    private:
+    void Finish()
+    {
+        if (sort)
+        {
+            std::vector<std::shared_ptr<TestOutput>> finalResults;
+            finalResults.reserve(cache.size());
+
+            for (auto x : cache)
+            {
+                finalResults.push_back(x.second);
+            }
+
+            std::sort(finalResults.begin(), finalResults.end(),
+                [](const std::shared_ptr<TestOutput> &lhs, const std::shared_ptr<TestOutput> &rhs)
+                {
+                    if (lhs->TestDetails().Suite != rhs->TestDetails().Suite)
+                    {
+                        return lhs->TestDetails().Suite < rhs->TestDetails().Suite;
+                    }
+
+                    return lhs->TestDetails().Name < rhs->TestDetails().Name;
+                });
+
+            std::string curSuite = "";
+            for (auto result : finalResults)
+            {
+                if (group)
+                {
+                    if (curSuite != result->TestDetails().Suite)
+                    {
+                        curSuite = result->TestDetails().Suite;
+
+                        std::cout << TestOutput::Fragment(Color::Suite, "\n\n==========\n[ " + curSuite + " ]\n==========\n");
+                    }
+                }
+
+                result->Print(group);
+            }
+        }
+    }
+
+private:
     OutputCache cache;
     bool verbose;
-    };
+    bool sort;
+    bool group;
+};
 
-ConsoleReporter::ConsoleReporter(bool verbose)
-    : cache(new ReportCache(verbose))
+ConsoleReporter::ConsoleReporter(bool verbose, bool sort, bool group)
+    : cache(new ReportCache(verbose, sort, group))
 {
+    std::cout.sync_with_stdio(false);
 }
 
 void ConsoleReporter::ReportStart(const TestDetails &)
 
 void ConsoleReporter::ReportSkip(const TestDetails &testDetails, const std::string &reason)
 {
-    cache->Instant(Color::Skip, "\n[ Skipped ] ");
-
-    if (!testDetails.Suite.empty())
-    {
-        cache->Instant(Color::Suite, testDetails.Suite);
-        cache->Instant(Color::Separator, TestSeparator);
-    }
-
-    cache->Instant(Color::TestName, testDetails.ShortName + "\n");
-    cache->Instant(Color::FileAndLine, to_string(testDetails.LineInfo) + ": ");
-    cache->Instant(Color::Skip, reason);
-    cache->Instant(Color::Default, "\n");
+    cache->Skip(testDetails, reason);
 }
 
 void ConsoleReporter::ReportFinish(const TestDetails &testDetails, Time::Duration timeTaken)
 
 void ConsoleReporter::ReportAllTestsComplete(size_t testCount, size_t skipped, size_t failureCount, Time::Duration totalTime)
 {
+    cache->Finish();
+
     Color failColor = Color::Default;
     Color skipColor = Color::Default;
     if (failureCount > 0)

File xUnit++.console/ConsoleReporter.h

View file
 class ConsoleReporter : public IOutput
 {
 public:
-    ConsoleReporter(bool verbose);
+    ConsoleReporter(bool verbose, bool sort, bool group);
 
     virtual void ReportStart(const TestDetails &) override;
     virtual void ReportEvent(const TestDetails &testDetails, const TestEvent &evt) override;

File xUnit++.console/main.cpp

View file
             std::sort(activeTestIds.begin(), activeTestIds.end());
 
             std::unique_ptr<xUnitpp::IOutput> reporter(options.xmlOutput.empty() ?
-                (xUnitpp::IOutput *)new xUnitpp::ConsoleReporter(options.verbose) :
+                (xUnitpp::IOutput *)new xUnitpp::ConsoleReporter(options.verbose, options.sort, options.group) :
                 (xUnitpp::IOutput *)new xUnitpp::Utilities::XmlReporter(options.xmlOutput));
             totalFailures += testAssembly.FilteredTestsRunner(options.timeLimit, options.threadLimit, *reporter,
                 [&](const xUnitpp::TestDetails &testDetails)