Anonymous avatar Anonymous committed 958ca27

Added the option to ask for a summary CSV containing just the intervals for the mean and stddev for each test

Comments (0)

Files changed (4)

                            outlierVariance, noteOutliers)
 import Criterion.Config (Config(..), Plot(..), fromLJ)
 import Criterion.Environment (Environment(..))
-import Criterion.IO (note, prolix)
+import Criterion.IO (note, prolix, summary)
 import Criterion.Measurement (getTime, runForAtLeast, secs, time_)
 import Criterion.Plot (plotWith, plotKDE, plotTiming)
 import Criterion.Types (Benchmarkable(..), Benchmark(..), bench, bgroup)
 import Statistics.Sample (mean, stdDev)
 import Statistics.Types (Sample)
 import System.Mem (performGC)
+import Text.Printf (printf)
 
 -- | Run a single benchmark, and return timings measured when
 -- executing it.
                  Moderate -> "moderately inflated"
                  Severe -> "severely inflated"
   bs "mean" em
+  summary cfg ","
   bs "std dev" es
+  summary cfg "\n"
   noteOutliers cfg (classifyOutliers times)
   note cfg "variance introduced by outliers: %.3f%%\n" (v * 100)
   note cfg "variance is %s by outliers\n" wibble
   return times
   where bs :: String -> Estimate -> IO ()
-        bs d e = note cfg "%s: %s, lb %s, ub %s, ci %.3f\n" d
-                   (secs $ estPoint e)
-                   (secs $ estLowerBound e) (secs $ estUpperBound e)
-                   (estConfidenceLevel e)
+        bs d e = do note cfg "%s: %s, lb %s, ub %s, ci %.3f\n" d
+                      (secs $ estPoint e)
+                      (secs $ estLowerBound e) (secs $ estUpperBound e)
+                      (estConfidenceLevel e)
+                    summary cfg $ printf "%g,%g,%g" 
+                      (estPoint e)
+                      (estLowerBound e) (estUpperBound e)
 
 plotAll :: Config -> [(String, Sample)] -> IO ()
 plotAll cfg descTimes = forM_ descTimes $ \(desc,times) -> do
 runAndAnalyse p cfg env = plotAll cfg <=< go ""
   where go pfx (Benchmark desc b)
             | p desc'   = do note cfg "\nbenchmarking %s\n" desc'
+                             summary cfg (show desc' ++ ",") -- String will be quoted
                              x <- runAndAnalyseOne cfg env desc' b
                              if cfgPlotSameAxis `fromLJ` cfg
                                then return      [(desc',x)]

Criterion/Config.hs

     , cfgPrintExit    :: PrintExit   -- ^ Whether to print information and exit.
     , cfgResamples    :: Last Int    -- ^ Number of resamples to perform.
     , cfgSamples      :: Last Int    -- ^ Number of samples to collect.
+    , cfgSummaryFile  :: Last String -- ^ Filename of summary CSV
     , cfgVerbosity    :: Last Verbosity -- ^ Whether to run verbosely.
     } deriving (Eq, Read, Show, Typeable)
 
                 , cfgPrintExit    = Nada
                 , cfgResamples    = ljust (100 * 1000)
                 , cfgSamples      = ljust 100
+                , cfgSummaryFile  = mempty
                 , cfgVerbosity    = ljust Normal
                 }
 
               , cfgPrintExit    = mempty
               , cfgResamples    = mempty
               , cfgSamples      = mempty
+              , cfgSummaryFile  = mempty
               , cfgVerbosity    = mempty
               }
 
     , cfgPlotSameAxis = app cfgPlotSameAxis a b
     , cfgPrintExit    = app cfgPrintExit a b
     , cfgSamples      = app cfgSamples a b
+    , cfgSummaryFile  = app cfgSummaryFile a b
     , cfgResamples    = app cfgResamples a b
     , cfgVerbosity    = app cfgVerbosity a b
     }
     , note
     , printError
     , prolix
+    , summary
     ) where
 
-import Criterion.Config (Config, Verbosity(..), cfgVerbosity, fromLJ)
+import Criterion.Config (Config, Verbosity(..), cfgSummaryFile, cfgVerbosity, fromLJ)
+import Data.Monoid (getLast)
 import System.IO (stderr, stdout)
 import Text.Printf (HPrintfType, hPrintf)
 
 -- | Print an error message.
 printError :: (HPrintfType r) => String -> r
 printError msg = hPrintf stderr msg
+
+-- | Add to summary CSV (if applicable)
+summary :: Config -> String -> IO ()
+summary cfg msg = case getLast $ cfgSummaryFile cfg of
+  Just fn -> appendFile fn msg
+  Nothing -> return ()

Criterion/Main.hs

           "number of samples to collect"
  , Option ['t'] ["plot-timing"] (ReqArg (plot Timing) "TYPE")
           "plot timings"
+ , Option ['u'] ["summary"] (ReqArg (\s -> return $ mempty { cfgSummaryFile = ljust s }) "FILENAME")
+          "produce a summary CSV file of all the benchmark means and standard deviations"
  , Option ['V'] ["version"] (noArg mempty { cfgPrintExit = Version })
           "display version, then exit"
  , Option ['v'] ["verbose"] (noArg mempty { cfgVerbosity = ljust Verbose })
       note cfg "Benchmarks:\n"
       mapM_ (note cfg "  %s\n") (sort $ concatMap benchNames bs)
     else do
+      case getLast $ cfgSummaryFile cfg of
+        Just fn -> writeFile fn "Name,Mean,MeanLB,MeanUB,Stddev,StddevLB,StddevUB\n"
+        Nothing -> return ()
       env <- measureEnvironment cfg
       let shouldRun b = null args || any (`isPrefixOf` b) args
       runAndAnalyse shouldRun cfg env $ BenchGroup "" bs
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.