Commits

Stefan Saasen committed bb09689

Show the duration off all the clone operations, not only the clone variants

Comments (0)

Files changed (3)

gnuplot/generate-git-ops-plot.plot

 set format x "%d/%m\n%H:%M"
 set timefmt "%Y-%m-%d %H:%M:%S"
 set output "git-clone-duration.png"
-set title "Duration of clone and shallow clone operations (seconds)"
+set title "Duration of git operations"
+set ylabel "Duration (seconds)"
+set multiplot layout 3, 1 title
 
-# Date | Duration (cache hit) | Duration (cache miss) | Client IP | hostname | Provider
+# Date |
+# Clone duration (cache hit) | Clone duration (cache miss) |
+# Fetch (hit) | Fetch (miss) | Shallow Clone (hit) | Shallow Clone (miss) |
+# Push (hit) | Push (miss) | Ref adv (hit) | Ref adv (miss) | Client IP"
 plot    "clone-duration"        using 1:($2/1000)                      with lines title "Clone (cache hit)",\
-        "clone-duration"        using 1:($3/1000)                      with lines title "Clone (cache miss)"
+        "clone-duration"        using 1:($3/1000)                      with lines title "Clone (cache miss)", \
+        "clone-duration"        using 1:($6/1000)                      with lines title "Shallow Clone (cache hit)", \
+        "clone-duration"        using 1:($7/1000)                      with lines title "Shallow Clone (cache miss)"
+
+unset title
+unset xlabel
+plot    "clone-duration"        using 1:($4/1000)                      with lines title "Fetch (cache hit)", \
+        "clone-duration"        using 1:($5/1000)                      with lines title "Fetch (cache miss)", \
+        "clone-duration"        using 1:($10/1000)                     with lines title "Rev adv. (cache hit)", \
+        "clone-duration"        using 1:($11/1000)                     with lines title "Rev adv. (cache miss)"
+
+plot    "clone-duration"        using 1:($8/1000)                      with lines title "Push (cache hit)", \
+        "clone-duration"        using 1:($9/1000)                      with lines title "Push (cache miss)"
+
+
 

logparser/src/Main.hs

         mapM_ (\pd -> printf "%s|%d\n" (formatLogDate $ getLogDate pd) (getValue pd)) plotData
 
 generateCloneRequestDurations :: (Input -> [RequestDurationStat]) -> [FilePath] -> IO ()
-generateCloneRequestDurations f path = do
-        plotData <- liftM f $ toLines path
-        printf "# Date | Duration (cache hit) | Duration (cache miss) | Client IP\n"
-        mapM_ (\pd -> printf "%s|%d|%d|%s\n" (show $ getDurationDate pd) (getDurationHit pd) (getDurationMiss pd) (getClientIp pd)) plotData
+generateCloneRequestDurations g path = do
+        plotData <- liftM g $ toLines path
+        printf "# Date | Clone duration (cache hit) | Clone duration (cache miss) | Fetch (hit) | Fetch (miss) | Shallow Clone (hit) | Shallow Clone (miss) | Push (hit) | Push (miss) | Ref adv (hit) | Ref adv (miss) | Client IP\n"
+        mapM_ (\(RequestDurationStat date clientIp [cm,fm,sm,pm,rm] [c,f,s,p,r])
+                -> printf "%s|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%s\n" (show date) c cm f fm s sm p pm r rm clientIp) plotData
 
 parseAndPrint :: (Show a) => (Input -> a) -> [FilePath] -> IO ()
 parseAndPrint f path = print . f . L.lines =<< readFiles path

logparser/src/Stash/Log/GitOpsAnalyser.hs

 data RequestDurationStat = RequestDurationStat {
     getDurationDate             :: !LogDate
    ,getClientIp                 :: !String
-   ,getDurationHit              :: !Int
-   ,getDurationMiss             :: !Int
+   ,cacheMissDurations          :: ![Int]
+   ,cacheHitDurations           :: ![Int] -- clone, fetch, shallow clone, push, ref advertisement
 }
 
 -- | Parse and aggregate the log file input into a list of hourly GitOperationStats
 
 -- | Return the duration of clone (clone and shallow clone) operations
 cloneRequestDuration :: Input -> [RequestDurationStat]
-cloneRequestDuration rawLines = collectRequestDurations rawLines authenticatedClone
+cloneRequestDuration rawLines = collectRequestDurations rawLines authenticatedGitOp
 
 
 -- =================================================================================
 
-authenticatedClone :: LogLine -> Bool
-authenticatedClone line = isJust (getUsername line) && (isShallowClone line || isClone line)
-
-emptyStats :: GitOperationStats
-emptyStats = GitOperationStats "" zero zero
-            where zero = replicate 5 0
+authenticatedGitOp :: LogLine -> Bool
+authenticatedGitOp line = isJust (getUsername line)
 
 collectRequestDurations :: Input -> (LogLine -> Bool) -> [RequestDurationStat]
 collectRequestDurations rawLines p = map m $ filter f $ parseLogLines rawLines
         where clientIp line = head $ split "," (S.unpack $ getRemoteAdress line)
-              m line        = if isCacheHit line
-                              then RequestDurationStat (getDate line) (clientIp line) (getRequestDuration line) 0
-                              else RequestDurationStat (getDate line) (clientIp line) 0 (getRequestDuration line)
-              f line        = isOutgoingLogLine line && p line
+              ops           = [isClone, isFetch, isShallowClone, isPush, isRefAdvertisement]
+              f line        = isOutgoingLogLine line && p line && (or $ map (\g -> g line) ops)
+              m line        =  let  duration    = getRequestDuration line
+                                    zero        = replicate 5 0
+                                    inc op      = if op line then (+duration) else id
+                                    missOps     = map (inc . uncachedOperation) ops
+                                    hitOps      = map (inc . cachedOperation) ops
+                                    !misses     = zipWith id missOps zero
+                                    !hits       = zipWith id hitOps zero
+                               in RequestDurationStat (getDate line) (clientIp line) misses hits
 
+emptyStats :: GitOperationStats
+emptyStats = GitOperationStats "" zero zero
+            where zero = replicate 5 0
 
 analyseGitOperations' :: (LogDate -> LogDate -> Bool) -> (LogDate -> String) -> Input -> [GitOperationStats]
 analyseGitOperations' comp formatLogDate rawLines =