Commits

Stefan Saasen committed c6947ae

Gnuplot: Rename the script that runs the logparser and generates the Gnuplot graphs

Comments (0)

Files changed (13)

 The output can be used to generate graphs, either using the provided `gnuplot`
 scripts or by using the Confluence chart macro.
 
-The `regenerate-graphs.sh` shows how to run the logparser, pipe the output into
+The `generate-access-log-graphs.sh` script shows how to run the logparser, pipe the output into
 data files and generate gnuplot graphics as PNG images.
 
 
-    $> ./regenerate-graphs.sh '/data/stash-access-log/atlassian-stash-access-2012-09*.log*'
+    $> ./generate-access-log-graphs.sh '/data/stash-access-log/atlassian-stash-access-2012-09*.log*'
 
 Available Commands
 ==================

generate-access-log-graphs.sh

+#!/bin/bash
+
+set -e
+set -u
+
+rm -f *.png
+
+# Ensure that 'logparser' is in the PATH (e.g. run rebuild.sh or 'cabal copy')
+DATE=`date "+%Y-%m"`
+if [ "$#" -le "1" ]; then
+    LOG_FILE=${1:-"../access-logs/atlassian-stash-access-${DATE}*"}
+else
+    LOG_FILE=$@
+fi
+
+export GNUPLOT_LIB="gnuplot:."
+
+time logparser gitOperations ${LOG_FILE} +RTS -sstderr > plot-git-ops.dat
+gnuplot < gnuplot/access-logs/generate-git-ops-plot.plot
+
+time logparser gitDurations ${LOG_FILE} +RTS -sstderr > clone-duration.dat
+gnuplot < gnuplot/access-logs/generate-git-durations.plot
+
+time logparser maxConn ${LOG_FILE} +RTS -sstderr > plot-all.dat
+gnuplot < gnuplot/access-logs/generate-max-conn-plot.plot
+
+time logparser protocolStats ${LOG_FILE} +RTS -sstderr > protocol-stats.dat
+gnuplot < gnuplot/access-logs/generate-git-protocol.plot
+
+time logparser repositoryStats ${LOG_FILE} +RTS -sstderr > repository-stats.dat
+gnuplot < gnuplot/access-logs/repository-stats.plot

gnuplot/access-logs/generate-git-durations.plot

+load "shared.plot"
+
+# ========================================
+set output "git-clone-duration.png"
+
+# ========================================
+
+set format x "%d/%m\n%H:%M"
+set timefmt "%Y-%m-%d %H:%M:%S"
+set title "Duration of git operations"
+set multiplot layout 3, 1 title
+
+
+# Date |
+# Clone duration (cache hit) | Clone duration (cache miss) |
+# Fetch (hit) | Fetch (miss) | Shallow Clone (hit) | Shallow Clone (miss) |
+# Push (hit) | Push (miss) | Ref adv (hit) | Ref adv (miss) | Client IP"
+plot    "clone-duration.dat"        using 1:($2/1000)                      with lines title "Clone (cache hit)",\
+        "clone-duration.dat"        using 1:($3/1000)                      with lines title "Clone (cache miss)", \
+        "clone-duration.dat"        using 1:($6/1000)                      with lines title "Shallow Clone (cache hit)", \
+        "clone-duration.dat"        using 1:($7/1000)                      with lines title "Shallow Clone (cache miss)"
+
+unset title
+unset xlabel
+plot    "clone-duration.dat"        using 1:($5/1000)                      with lines title "Fetch", \
+        "clone-duration.dat"        using 1:($10/1000)                     with lines title "Rev adv. (cache hit)", \
+        "clone-duration.dat"        using 1:($11/1000)                     with lines title "Rev adv. (cache miss)"
+
+plot    "clone-duration.dat"        using 1:($9/1000)                      with lines title "Push"
+
+
+unset multiplot
+set output "git-clone-duration-distribution.png"
+
+set title "Distribution of git clone operations"
+plot    "clone-duration.dat"        using 1:(($2+$3)/1000)                      with points pointtype 6 title "Clone",\
+        "clone-duration.dat"        using 1:(($6+$7)/1000)                      with points pointtype 6 title "Shallow Clone"

gnuplot/access-logs/generate-git-ops-plot.plot

+load "shared.plot"
+
+
+# ========================================
+set output "git-ops.png"
+
+# ========================================
+set title "Git operations per hour (stacked)"
+#  0: Date
+#  1-6: clone, fetch, shallow clone, push, ref advertisement (sum cache hit & cache miss)
+#  7-11: cache hits
+#  12-16: cache miss'
+plot    "plot-git-ops.dat" using 1:($2+$3+$4+$5) with filledcurves x1 lt rgb "#80522D" title "push", \
+        "plot-git-ops.dat" using 1:($2+$3+$4)    with filledcurves x1 lt rgb "#D9C293" title "shallow clone", \
+        "plot-git-ops.dat" using 1:($2+$3)       with filledcurves x1 lt rgb "#DED9D9" title "fetch", \
+        "plot-git-ops.dat" using 1:2             with filledcurves x1 lt rgb "#253B57" title "clone"
+
+# ========================================
+
+set output "git-ops-lines.png"
+set title "Git operations per hour"
+plot    "plot-git-ops.dat" using 1:5 with lines title "push", \
+        "plot-git-ops.dat" using 1:4 with lines title "shallow clone", \
+        "plot-git-ops.dat" using 1:3 with lines title "fetch", \
+        "plot-git-ops.dat" using 1:2 with lines title "clone"
+
+# ===================================================================================
+
+set output "git-ops-caching.png"
+plot    "plot-git-ops.dat" using 1:7  with lines title "clone (hit)", \
+        "plot-git-ops.dat" using 1:9  with lines title "shallow clone (hit)", \
+        "plot-git-ops.dat" using 1:12 with lines title "clone (miss)", \
+        "plot-git-ops.dat" using 1:14 with lines title "shallow clone (miss)"
+
+# ===================================================================================
+
+set output "git-ops-ref-advertisement.png"
+plot    "plot-git-ops.dat" using 1:6  with lines title "ref advertisement", \
+        "plot-git-ops.dat" using 1:11 with lines title "ref advertisement (hit)", \
+        "plot-git-ops.dat" using 1:16 with lines title "ref advertisement (miss)"
+

gnuplot/access-logs/generate-git-protocol.plot

+load "shared.plot"
+
+# ========================================
+set output "git-protocol.png"
+# ========================================
+
+set ylabel "Protocol usage"
+set title "Git protocol usage per hour"
+set ytics nomirror
+
+set multiplot layout 2, 1 title
+
+# ========================================
+
+plot    "protocol-stats.dat" using 1:2 with lines title "ssh", \
+        "protocol-stats.dat" using 1:3 with lines title "http(s)"
+
+set logscale y
+set yrange [1:*]
+set title "Git protocol usage per hour (logscale)"
+plot    "protocol-stats.dat" using 1:2 with lines title "ssh", \
+        "protocol-stats.dat" using 1:3 with lines title "http(s)"

gnuplot/access-logs/generate-max-conn-plot.plot

+load "shared.plot"
+
+set output "max-conn.png"
+set ylabel "Connections"
+set title "Concurrent Connections per hour"
+plot "plot-all.dat" using 1:2 with lines lw 2 lt 3 title "max concurrent connections"

gnuplot/access-logs/repository-stats.plot

+set datafile separator "|"
+set terminal png size 1400,1000
+
+set xlabel "Repository"
+
+set grid
+set auto x
+set xtic out nomirror rotate by -45 font ",8"
+
+
+set output "repository-stats.png"
+set ylabel "Repository - Clone statistics"
+set title "Number of clones per repository for the whole timeframe"
+
+plot "repository-stats.dat" every ::::29 using 2:xticlabels(1) with lines title "Number of clones"
+

gnuplot/generate-git-durations.plot

-load "shared.plot"
-
-# ========================================
-set output "git-clone-duration.png"
-
-# ========================================
-
-set format x "%d/%m\n%H:%M"
-set timefmt "%Y-%m-%d %H:%M:%S"
-set title "Duration of git operations"
-set multiplot layout 3, 1 title
-
-
-# Date |
-# Clone duration (cache hit) | Clone duration (cache miss) |
-# Fetch (hit) | Fetch (miss) | Shallow Clone (hit) | Shallow Clone (miss) |
-# Push (hit) | Push (miss) | Ref adv (hit) | Ref adv (miss) | Client IP"
-plot    "clone-duration.dat"        using 1:($2/1000)                      with lines title "Clone (cache hit)",\
-        "clone-duration.dat"        using 1:($3/1000)                      with lines title "Clone (cache miss)", \
-        "clone-duration.dat"        using 1:($6/1000)                      with lines title "Shallow Clone (cache hit)", \
-        "clone-duration.dat"        using 1:($7/1000)                      with lines title "Shallow Clone (cache miss)"
-
-unset title
-unset xlabel
-plot    "clone-duration.dat"        using 1:($5/1000)                      with lines title "Fetch", \
-        "clone-duration.dat"        using 1:($10/1000)                     with lines title "Rev adv. (cache hit)", \
-        "clone-duration.dat"        using 1:($11/1000)                     with lines title "Rev adv. (cache miss)"
-
-plot    "clone-duration.dat"        using 1:($9/1000)                      with lines title "Push"
-
-
-unset multiplot
-set output "git-clone-duration-distribution.png"
-
-set title "Distribution of git clone operations"
-plot    "clone-duration.dat"        using 1:(($2+$3)/1000)                      with points pointtype 6 title "Clone",\
-        "clone-duration.dat"        using 1:(($6+$7)/1000)                      with points pointtype 6 title "Shallow Clone"

gnuplot/generate-git-ops-plot.plot

-load "shared.plot"
-
-
-# ========================================
-set output "git-ops.png"
-
-# ========================================
-set title "Git operations per hour (stacked)"
-#  0: Date
-#  1-6: clone, fetch, shallow clone, push, ref advertisement (sum cache hit & cache miss)
-#  7-11: cache hits
-#  12-16: cache miss'
-plot    "plot-git-ops.dat" using 1:($2+$3+$4+$5) with filledcurves x1 lt rgb "#80522D" title "push", \
-        "plot-git-ops.dat" using 1:($2+$3+$4)    with filledcurves x1 lt rgb "#D9C293" title "shallow clone", \
-        "plot-git-ops.dat" using 1:($2+$3)       with filledcurves x1 lt rgb "#DED9D9" title "fetch", \
-        "plot-git-ops.dat" using 1:2             with filledcurves x1 lt rgb "#253B57" title "clone"
-
-# ========================================
-
-set output "git-ops-lines.png"
-set title "Git operations per hour"
-plot    "plot-git-ops.dat" using 1:5 with lines title "push", \
-        "plot-git-ops.dat" using 1:4 with lines title "shallow clone", \
-        "plot-git-ops.dat" using 1:3 with lines title "fetch", \
-        "plot-git-ops.dat" using 1:2 with lines title "clone"
-
-# ===================================================================================
-
-set output "git-ops-caching.png"
-plot    "plot-git-ops.dat" using 1:7  with lines title "clone (hit)", \
-        "plot-git-ops.dat" using 1:9  with lines title "shallow clone (hit)", \
-        "plot-git-ops.dat" using 1:12 with lines title "clone (miss)", \
-        "plot-git-ops.dat" using 1:14 with lines title "shallow clone (miss)"
-
-# ===================================================================================
-
-set output "git-ops-ref-advertisement.png"
-plot    "plot-git-ops.dat" using 1:6  with lines title "ref advertisement", \
-        "plot-git-ops.dat" using 1:11 with lines title "ref advertisement (hit)", \
-        "plot-git-ops.dat" using 1:16 with lines title "ref advertisement (miss)"
-

gnuplot/generate-git-protocol.plot

-load "shared.plot"
-
-# ========================================
-set output "git-protocol.png"
-# ========================================
-
-set ylabel "Protocol usage"
-set title "Git protocol usage per hour"
-set ytics nomirror
-
-set multiplot layout 2, 1 title
-
-# ========================================
-
-plot    "protocol-stats.dat" using 1:2 with lines title "ssh", \
-        "protocol-stats.dat" using 1:3 with lines title "http(s)"
-
-set logscale y
-set yrange [1:*]
-set title "Git protocol usage per hour (logscale)"
-plot    "protocol-stats.dat" using 1:2 with lines title "ssh", \
-        "protocol-stats.dat" using 1:3 with lines title "http(s)"

gnuplot/generate-max-conn-plot.plot

-load "shared.plot"
-
-set output "max-conn.png"
-set ylabel "Connections"
-set title "Concurrent Connections per hour"
-plot "plot-all.dat" using 1:2 with lines lw 2 lt 3 title "max concurrent connections"

gnuplot/repository-stats.plot

-set datafile separator "|"
-set terminal png size 1400,1000
-
-set xlabel "Repository"
-
-set grid
-set output "repository-stats.png"
-set auto x
-set ylabel "Repository - Clone statistics"
-set title "Number of clones per repository for the whole timeframe"
-
-set xtic out nomirror rotate by -45 font ",8"
-
-plot "repository-stats.dat" every ::::29 using 2:xticlabels(1) with lines title "Number of clones"
-

regenerate-graphs.sh

-#!/bin/bash
-
-set -e
-set -u
-
-rm -f *.png
-
-# Ensure that 'logparser' is in the PATH (e.g. run rebuild.sh or 'cabal copy')
-DATE=`date "+%Y-%m"`
-if [ "$#" -le "1" ]; then
-    LOG_FILE=${1:-"../access-logs/atlassian-stash-access-${DATE}*"}
-else
-    LOG_FILE=$@
-fi
-
-export GNUPLOT_LIB="gnuplot:."
-
-time logparser gitOperations ${LOG_FILE} +RTS -sstderr > plot-git-ops.dat
-gnuplot < gnuplot/generate-git-ops-plot.plot
-
-time logparser gitDurations ${LOG_FILE} +RTS -sstderr > clone-duration.dat
-gnuplot < gnuplot/generate-git-durations.plot
-
-time logparser maxConn ${LOG_FILE} +RTS -sstderr > plot-all.dat
-gnuplot < gnuplot/generate-max-conn-plot.plot
-
-time logparser protocolStats ${LOG_FILE} +RTS -sstderr > protocol-stats.dat
-gnuplot < gnuplot/generate-git-protocol.plot
-
-time logparser repositoryStats ${LOG_FILE} +RTS -sstderr > repository-stats.dat
-gnuplot < gnuplot/repository-stats.plot