Commits

Bryan O'Sullivan committed ff4ba18

Make outlier data available to the reporting code.

Comments (0)

Files changed (4)

 
 import Control.Monad ((<=<), replicateM_, when)
 import Control.Monad.Trans (liftIO)
-import Criterion.Analysis (OutlierEffect(..), OutlierVariance(..),
+import Criterion.Analysis (Outliers(..), OutlierEffect(..), OutlierVariance(..),
                            SampleAnalysis(..), analyseSample,
                            classifyOutliers, noteOutliers)
 import Criterion.Config (Config(..), Verbosity(..), fromLJ)
 
 -- | Run a single benchmark and analyse its performance.
 runAndAnalyseOne :: Benchmarkable b => Environment -> String -> b
-                 -> Criterion (Sample,SampleAnalysis)
+                 -> Criterion (Sample,SampleAnalysis,Outliers)
 runAndAnalyseOne env _desc b = do
   times <- runBenchmark env b
   ci <- getConfigItem $ fromLJ cfgConfInterval
   numResamples <- getConfigItem $ fromLJ cfgResamples
   _ <- prolix "analysing with %d resamples\n" numResamples
   an@SampleAnalysis{..} <- liftIO $ analyseSample ci times numResamples
-  let OutlierVariance{..} = anOutliers
+  let OutlierVariance{..} = anOutlierVar
   let wibble = case ovEffect of
                  Unaffected -> "unaffected" :: String
                  Slight -> "slightly inflated"
   bs "std dev" anStdDev
   summary "\n"
   vrb <- getConfigItem $ fromLJ cfgVerbosity
+  let out = classifyOutliers times
   when (vrb == Verbose || (ovEffect > Unaffected && vrb > Quiet)) $ do
-    noteOutliers (classifyOutliers times)
+    noteOutliers out
     _ <- note "variance introduced by outliers: %.3f%%\n" (ovFraction * 100)
     _ <- note "variance is %s by outliers\n" wibble
     return ()
-  return (times,an)
+  return (times,an,out)
   where bs :: String -> Estimate -> Criterion ()
         bs d e = do _ <- note "%s: %s, lb %s, ub %s, ci %.3f\n" d
                       (secs $ estPoint e)
                       (estPoint e)
                       (estLowerBound e) (estUpperBound e)
 
-plotAll :: [(String, Sample, SampleAnalysis)] -> Criterion ()
+plotAll :: [(String, Sample, SampleAnalysis, Outliers)] -> Criterion ()
 plotAll descTimes = do
-  report "foo" (zipWith (\n (d,t,a) -> Report n d t a) [0..] descTimes)
+  report "foo" (zipWith (\n (d,t,a,o) -> Report n d t a o) [0..] descTimes)
 
 -- | Run, and analyse, one or more benchmarks.
 runAndAnalyse :: (String -> Bool) -- ^ A predicate that chooses
   where go pfx (Benchmark desc b)
             | p desc'   = do _ <- note "\nbenchmarking %s\n" desc'
                              summary (show desc' ++ ",") -- String will be quoted
-                             (x,an) <- runAndAnalyseOne env desc' b
-                             return [(desc',x,an)]
+                             (x,an,out) <- runAndAnalyseOne env desc' b
+                             return [(desc',x,an,out)]
             | otherwise = return []
             where desc' = prefix pfx desc
         go pfx (BenchGroup desc bs) =

Criterion/Analysis.hs

   return SampleAnalysis {
                anMean = estMean
              , anStdDev = estStdDev
-             , anOutliers = ov
+             , anOutlierVar = ov
              }
 
 -- | Display a report of the 'Outliers' present in a 'Sample'.

Criterion/Analysis/Types.hs

 {-# INLINE addOutliers #-}
 
 -- | Analysis of the extent to which outliers in a sample affect its
--- mean and standard deviation.
+-- standard deviation (and to some extent, its mean).
 data OutlierVariance = OutlierVariance {
       ovEffect   :: OutlierEffect
     -- ^ Qualitative description of effect.
     } deriving (Eq, Read, Show, Typeable, Data)
 
 instance NFData OutlierVariance where
-    rnf OutlierVariance{..} = rnf ovEffect `seq` rnf ovFraction `seq` ()
+    rnf OutlierVariance{..} = rnf ovEffect `seq` rnf ovFraction
 
 -- | Result of a bootstrap analysis of a non-parametric sample.
 data SampleAnalysis = SampleAnalysis {
       anMean :: B.Estimate
     , anStdDev :: B.Estimate
-    , anOutliers :: OutlierVariance
+    , anOutlierVar :: OutlierVariance
     } deriving (Eq, Show, Typeable, Data)
 
 instance NFData SampleAnalysis where
     rnf SampleAnalysis{..} =
-        rnf anMean `seq` rnf anStdDev `seq` rnf anOutliers `seq` ()
+        rnf anMean `seq` rnf anStdDev `seq` rnf anOutlierVar

Criterion/Report.hs

     ) where
 
 import Control.Monad.IO.Class (liftIO)
-import Criterion.Analysis (SampleAnalysis(..))
+import Criterion.Analysis (Outliers(..), SampleAnalysis(..))
 import Criterion.Monad (Criterion)
 import Data.ByteString.Char8 ()
 import Data.Char (isSpace, toLower)
     , reportName :: String
     , reportTimes :: Sample
     , reportAnalysis :: SampleAnalysis
+    , reportOutliers :: Outliers
     } deriving (Eq, Show, Typeable, Data)
 
 templateDir :: FilePath
                            "times"    -> enc reportTimes
                            "kdetimes" -> enc kdeTimes
                            "kdepdf"   -> enc kdePDF
-                           _          -> mkGenericContext reportAnalysis $
+                           ('a':'n':_)-> mkGenericContext reportAnalysis $
+                                         H.encodeStr nym
+                           _          -> mkGenericContext reportOutliers $
                                          H.encodeStr nym
           where (kdeTimes,kdePDF) = kde 128 reportTimes
       enc :: (A.ToJSON a) => a -> MuType m