Commits

Chris Doble committed 0f7782a

Detect flakey tests.

Comments (0)

Files changed (4)

src/main/java/com/atlassian/bamboo/plugins/failureleaderboard/TestCaseFailures.java

         }
     }
 
+    private boolean isFlakey;
     private TestCase testCase;
     private Set<TestCaseResult> failedTestCaseResults;
 
 
     @Override
     public int compareTo(TestCaseFailures testCaseFailures) {
-        return failedTestCaseResults.size() - testCaseFailures.getFailedTestCaseResults().size();
+        return testCaseFailures.getFailedTestCaseResults().size() - getFailedTestCaseResults().size();
+    }
+
+    public boolean isFlakey() {
+        return isFlakey;
+    }
+
+    public void setIsFlakey(boolean isFlakey) {
+        this.isFlakey = isFlakey;
     }
 
     public TestCase getTestCase() {

src/main/java/com/atlassian/bamboo/plugins/failureleaderboard/ViewFailureLeaderboard.java

 import com.atlassian.bamboo.chains.ChainResultsSummary;
 import com.atlassian.bamboo.resultsummary.BuildResultsSummary;
 import com.atlassian.bamboo.resultsummary.ResultsSummary;
-import com.atlassian.bamboo.resultsummary.tests.TestCaseResult;
+import com.atlassian.bamboo.resultsummary.tests.*;
 import com.atlassian.bamboo.ww2.actions.ChainActionSupport;
 import com.atlassian.bamboo.ww2.aware.ResultsListAware;
 import com.google.common.collect.Lists;
  */
 public class ViewFailureLeaderboard extends ChainActionSupport implements ResultsListAware {
     private FilterController filterController;
+    private List<TestCaseFailures> leadingTestCaseFailures;
     private List<ResultsSummary> resultsList;
 
-    public List<TestCaseFailures> getLeadingTestCases() {
-        List<TestCaseFailures> allTestCaseFailures = new ArrayList<TestCaseFailures>();
+    /**
+     * Determine if a build appears to be flakey.
+     *
+     * @param resultsSummary The build.
+     * @return {@code true} iff the build appears to be flakey.
+     */
+    private boolean buildIsFlakey(ResultsSummary resultsSummary) {
+        TestResultsSummary testResults = resultsSummary.getTestResultsSummary();
+        int changedTests = testResults.getFixedTestCaseCount() + testResults.getNewFailedTestCaseCount();
 
-        for (TestCaseResult testCaseResult : getTestCaseResults()) {
-            TestCaseFailures testCaseFailures = null;
-            for (TestCaseFailures i : allTestCaseFailures) {
-                if (i.getTestCase().equals(testCaseResult.getTestCase())) {
-                    testCaseFailures = i;
-                    break;
-                }
-            }
+        // If no code was changed, but test state changed (fixed / started failing), then it's flakey.
+        return !resultsSummary.hasChanges() && changedTests > 0;
+    }
+
+    public List<TestCaseFailures> getLeadingTestCaseFailures() {
+        if (leadingTestCaseFailures == null) {
+            // A map of TestCase IDs to associated TestCaseFailure objects.
+            Map<Long, TestCaseFailures> testCaseMap = new HashMap<Long, TestCaseFailures>();
 
-            if (testCaseFailures == null) {
-                testCaseFailures = new TestCaseFailures(testCaseResult.getTestCase());
-                allTestCaseFailures.add(testCaseFailures);
+            for (ChainResultsSummary resultsSummary : getInterestingBuilds()) {
+                extractFailedTests(resultsSummary, testCaseMap);
+
+                if (buildIsFlakey(resultsSummary)) {
+                    extractFlakeyTests(resultsSummary, testCaseMap);
+                }
             }
 
-            testCaseFailures.getFailedTestCaseResults().add(testCaseResult);
+            leadingTestCaseFailures = new ArrayList<TestCaseFailures>(testCaseMap.values());
+            Collections.sort(leadingTestCaseFailures);
         }
 
-        Collections.sort(allTestCaseFailures);
-        Collections.reverse(allTestCaseFailures);
-        return allTestCaseFailures;
+        return leadingTestCaseFailures;
     }
 
     /**
-     * Retrieves all failed test case results from the results list.
+     * Extracts test case failures from a build and inserts them into a test case map.
      *
-     * @return All failed test case results from the results list.
+     * @param resultsSummary The build.
+     * @param testCaseMap The test case map.
      */
-    private List<TestCaseResult> getTestCaseResults() {
-        List<TestCaseResult> testCaseResults = new ArrayList<TestCaseResult>();
-        for (ResultsSummary buildSummary : getResultsList()) {
-            // Is it safe to cast? We do subclass ChainActionSupport after all...
-            for (BuildResultsSummary jobResult : ((ChainResultsSummary)buildSummary).getFailedJobResults()) {
-                testCaseResults.addAll(jobResult.getFilteredTestResults().getAllFailedTestList());
+    private void extractFailedTests(ChainResultsSummary resultsSummary, Map<Long, TestCaseFailures> testCaseMap) {
+        for (BuildResultsSummary jobResult : resultsSummary.getFailedJobResults()) {
+            for (TestCaseResult testCaseResult : jobResult.getFilteredTestResults().getAllFailedTestList()) {
+                TestCaseFailures testCaseFailures = getTestCaseFailures(testCaseResult.getTestCase(), testCaseMap);
+                testCaseFailures.getFailedTestCaseResults().add(testCaseResult);
             }
         }
+    }
 
-        return testCaseResults;
+    /**
+     * Identify flakey tests in a build and update a test case map accordingly.
+     *
+     * Assumes that no code changes were made in {@code resultsSummary}.
+     *
+     * @param resultsSummary The build.
+     * @param testCaseMap The test case map.
+     */
+    private void extractFlakeyTests(ChainResultsSummary resultsSummary, Map<Long, TestCaseFailures> testCaseMap) {
+        for (ResultsSummary jobResult : resultsSummary.getOrderedJobResultSummaries()) {
+            // Extract the test cases whose state changed (fixed / started failing). These are flakey!
+            FilteredTestResults<TestClassResult> testResults = ((BuildResultsSummary)jobResult).getFilteredTestResults();
+            List<TestCaseResult> changedTests = new ArrayList<TestCaseResult>(testResults.getFixedTests().values());
+            changedTests.addAll(testResults.getNewFailedTests().values());
+
+            for (TestCaseResult testCaseResult : changedTests) {
+                getTestCaseFailures(testCaseResult.getTestCase(), testCaseMap).setIsFlakey(true);
+            }
+        }
     }
 
     public FilterController getFilterController() {
         return filterController;
     }
 
+    /**
+     * Extracts interesting builds from {@code resultsList}.
+     *
+     * An "interesting" build is one that failed or one where the state of the
+     * test cases changed, but no code changes were made (a sign of flakiness).
+     *
+     * @return A list of the interesting builds in {@code resultsList}.
+     */
+    private List<ChainResultsSummary> getInterestingBuilds() {
+        List<ChainResultsSummary> interestingBuilds = new ArrayList<ChainResultsSummary>();
+
+        for (ResultsSummary resultsSummary : resultsList) {
+            TestResultsSummary testResults = resultsSummary.getTestResultsSummary();
+            if (testResults.hasFailedTestResults() || buildIsFlakey(resultsSummary)) {
+                interestingBuilds.add((ChainResultsSummary)resultsSummary);
+            }
+        }
+
+        return interestingBuilds;
+    }
+
     @Override
     public List<? extends ResultsSummary> getResultsList() {
         return resultsList;
     public void setResultsList(List<? extends ResultsSummary> resultsList) {
         this.resultsList = Lists.newArrayList(resultsList);
     }
+
+    /**
+     * Retrieve or create a {@code TestCaseFailures} object for a {@code TestCase}.
+     *
+     * @param testCase The test case.
+     * @param testCaseMap The test case map.
+     * @return A {@code TestCaseFailures} object associated with {@code TestCase}.
+     */
+    private TestCaseFailures getTestCaseFailures(TestCase testCase, Map<Long, TestCaseFailures> testCaseMap) {
+        Long testCaseId = testCase.getId();
+        TestCaseFailures testCaseFailures = testCaseMap.get(testCaseId);
+
+        if (testCaseFailures == null) {
+            testCaseFailures = new TestCaseFailures(testCase);
+            testCaseMap.put(testCaseId, testCaseFailures);
+        }
+
+        return testCaseFailures;
+    }
 }

src/main/resources/css/styles.css

     margin-bottom: 0.1em;
 }
 
-div.failure-leaderboard table span.quarantined {
-    background-color: #5391DF;
+div.failure-leaderboard span.pill {
     color: #FFF;
     -moz-border-radius: 3px;
     -webkit-border-radius: 3px;
     text-transform: uppercase;
 }
 
+div.failure-leaderboard table span.pill.flakey {
+    background-color: #900;
+    padding-right: 4px;
+}
+
+div.failure-leaderboard table span.pill.quarantined {
+    background-color: #5391DF;
+}
+
 div.failure-leaderboard table tr.builds {
     background-color: #F7F7F7;
 }
     display: none;
 }
 
-div.failure-leaderboard table a.build-number {
+div.failure-leaderboard table tr.builds ol {
+    font-size: 0;
+    padding: 0;
+}
+
+div.failure-leaderboard table tr.builds ol li {
+    display: inline;
+}
+
+div.failure-leaderboard table tr.builds ol li a {
+    font-size: 13px;
     font-weight: bold;
-    margin-right: 0.25em;
+    margin-right: 0.5em;
 }

src/main/resources/templates/viewFailureLeaderboard.ftl

     <h1>Failure Leaderboard</h1>
 
     <div class="failure-leaderboard">
-        [#assign allTestCaseFailures = leadingTestCases/]
-        [#if allTestCaseFailures.size() > 0]
+        [#if leadingTestCaseFailures?has_content]
             <div class="twixie-controls">
                 <a href="#" class="expand-all"><span class="icon icon-expand"></span>Expand all</a>
                 <a href="#" class="collapse-all"><span class="icon icon-collapse"></span>Collapse all</a>
                 </thead>
                 <tbody>
                     [#assign buildCount = resultsList.size()/]
-                    [#list allTestCaseFailures as testCaseFailures]
+                    [#list leadingTestCaseFailures as testCaseFailures]
                         <tr>
                             [#assign testCase = testCaseFailures.testCase/]
                             [#assign failureRate = testCaseFailures.failedTestCaseResults.size() * 100.0 / buildCount/]
                             <td class="failure-rate">${failureRate?string("0")}%</td>
                             <td>
                                 <p class="test-name">${testCase.testClass.shortName} ${testCase.name}</p>
+                                [#if testCaseFailures.isFlakey()]
+                                    <span class="pill flakey">Flakey</span>
+                                [/#if]
                                 [#if testCase.isQuarantined()]
-                                    <span class="quarantined">Quarantined</span>
+                                    <span class="pill quarantined">Quarantined</span>
                                 [/#if]
                             </td>
                         </tr>
                         <tr class="builds collapsed">
                             <td colspan="3">
+                                <ol>
                                 [#list testCaseFailures.failedTestCaseResults as testCaseResult]
                                     [#assign buildKey = testCaseResult.testCase.testClass.plan.key/]
                                     [#assign buildNumber = testCaseResult.testClassResult.buildResultsSummary.buildNumber/]
-                                    <a class="build-number" href="[@ww.url value=fn.getTestCaseResultUrl(buildKey, buildNumber, testCaseResult.testCase.id)/]">#${buildNumber}</a>
+                                    <li><a class="build-number" href="[@ww.url value=fn.getTestCaseResultUrl(buildKey, buildNumber, testCaseResult.testCase.id)/]">#${buildNumber}</a></li>
                                 [/#list]
+                                </ol>
                             </td>
                         </tr>
                     [/#list]
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.