Commits

Christian Theune  committed e2bc1cc

Towards configurable sets of graphs.

- Fix bug in smoothie.js to allow enabling/disabling resetBounds
- add a new view that describes charts and timelines
- automatically set up the monitor page based on the new view
- get a nice color palette
- structure haproxy into a source which fetches data and measures which analyze the data
- fix hanging browser by using gevent.sleep instead of time.sleep in websocket generator
- add test runner and debug middleware

  • Participants
  • Parent commits bc1a734

Comments (0)

Files changed (9)

File buildout.cfg

 [buildout]
-parts = livemonitor
+parts = livemonitor test
 develop = .
 
 [livemonitor]
 recipe = zc.recipe.egg
 eggs = livemonitor
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = livemonitor
+defaults = ['--auto-color']
         'distribute',
         'flask',
         'gevent-websocket',
+        'repoze.debug',
     ],
     entry_points="""
         [console_scripts]

File src/livemonitor/app.py

-from .haproxy import HAProxy
 from flask import Flask, render_template, make_response, request
 from gevent.pywsgi import WSGIServer
 from geventwebsocket.handler import WebSocketHandler
+from repoze.debug.responselogger import ResponseLoggingMiddleware
+import gevent
 import json
+import livemonitor.haproxy
 import logging
 import random
 import threading
     return make_response()
 
 
+
+##### UI configuration
+
+charts = []
+@app.route('/charts')
+def get_charts():
+    return json.dumps([
+        ["RequestRate", "ErrorResponses", "Error4xx", "Error5xx"],
+        ["Health"]])
+
+
+##### Data streaming
+
 @app.route('/data')
 def data():
     if 'wsgi.websocket' in request.environ:
 
 def data_one():
     data =  {}
-    for source in sources:
-        for name, value, time in source.metrics():
-            data[name] = dict(value=value, time=time)
+    for measure in measures:
+        data[measure.__class__.__name__] = dict(
+            value=measure.value, time=int(measure.timestamp*1000))
     return json.dumps(data)
 
 
     while True:
         data = data_one()
         ws.send(data)
-        time.sleep(0.1)
+        gevent.sleep(0.1)
 
 
 sources = []
-
+measures = []
 
 def update():
     while True:
         for source in sources:
             source.update()
+        for measure in measures:
+            measure.update()
         time.sleep(0.1)
 
 
 def main():
-    sources.append(HAProxy())
-    app.debug = True
+    haproxy = livemonitor.haproxy.configure()
+    sources.append(haproxy[0])
+    measures.extend(haproxy[1])
+
     update_thread = threading.Thread(target=update)
     update_thread.setDaemon(True)
     update_thread.start()
-    server = WSGIServer(('', 5000), app, handler_class=WebSocketHandler)
+
+    logging.basicConfig(level=logging.DEBUG)
+
+    pipeline = ResponseLoggingMiddleware(
+        app,
+        max_bodylen=3072,
+        keep=100,
+        verbose_logger=logging.getLogger('verbose'),
+        trace_logger=logging.getLogger('trace'))
+    server = WSGIServer(('', 5000), pipeline, handler_class=WebSocketHandler)
     server.serve_forever()

File src/livemonitor/haproxy.py

 import csv
+import livemonitor.measures
 import time
 import urllib2
 
 
-class HAProxy(object):
+class Source(object):
 
     url = 'http://localhost:8092/admin/stats;csv'
 
-    last_update = 0
-    _errors = 0
-    errors = 0
-    frontend_request_rate = 0
-
     def update(self):
         data = urllib2.urlopen(self.url)
         reader = csv.DictReader(data, delimiter=',')
-        errors = 0
-        for stat in reader:
-            if stat['svname'] == 'FRONTEND':
-                self.request_rate = int(stat['req_rate'])
-            if stat['svname'] == 'BACKEND':
-                errors += int(stat['eresp'])
-                errors += int(stat['hrsp_4xx'])
-                errors += int(stat['hrsp_5xx'])
+        self.status = list(reader)
+        self.timestamp = time.time()
 
-        update = time.time() * 1000
-        self.errors = (errors-self._errors) / ((update-self.last_update)/1000)
-        self._errors = errors
+    def _filter(self, include=(), exclude=()):
+        values = []
+        assert not (include and exclude)
+        for line in self.status:
+            value = line['svname']
+            if include and value in include:
+                values.append(line)
+            if exclude and value not in exclude:
+                values.append(line)
+        return values
 
-        self.last_update = update
+    def get_frontend(self):
+        return self._filter(include=('FRONTEND',))[0]
 
-    def metrics(self):
-        yield ('haproxy_requests', self.request_rate, self.last_update)
-        yield ('haproxy_errors', self.errors, self.last_update)
+    def get_backend(self):
+        return self._filter(include=('BACKEND',))[0]
+
+    def get_servers(self):
+        return self._filter(exclude=('BACKEND', 'FRONTEND'))
+
+
+class RequestRate(livemonitor.measures.Measure):
+
+    def update(self):
+        frontend = self.source.get_frontend()
+        self.value = int(frontend['req_rate'])
+        self.timestamp = self.source.timestamp
+
+
+class ErrorRateBase(livemonitor.measures.Measure):
+
+    field = None
+
+    value = 0
+    timestamp = 0
+    last_absolute = None
+
+    def update(self):
+        backend = self.source.get_backend()
+        errors = int(backend[self.field])
+        if self.last_absolute is None:
+            self.last_absolute = errors
+            self.timestamp = self.source.timestamp
+            return
+        self.value = (errors - self.last_absolute) / (self.source.timestamp - self.timestamp)
+        self.last_absolute = errors
+        self.timestamp = self.source.timestamp
+
+
+class ErrorResponses(ErrorRateBase):
+    field = 'eresp'
+
+
+class Error4xx(ErrorRateBase):
+    field = 'hrsp_4xx'
+
+
+class Error5xx(ErrorRateBase):
+    field = 'hrsp_5xx'
+
+
+class Health(livemonitor.measures.Measure):
+
+    def update(self):
+        total = 0
+        alive = 0
+        for server in self.source.get_servers():
+            total += 1
+            if server['status'] == 'UP':
+                alive += 1
+        self.value = alive / total * 100
+        self.timestamp = self.source.timestamp
+
+
+def configure():
+    source = Source()
+    request_rate = RequestRate(source)
+    health = Health(source)
+    error_responses = ErrorResponses(source)
+    error_4xx = Error4xx(source)
+    error_5xx = Error5xx(source)
+    return (source, (request_rate, health, error_responses, error_4xx, error_5xx))

File src/livemonitor/measures.py

+
+
+class Measure(object):
+
+    value = None
+
+    def __init__(self, source):
+        self.source = source
+
+    def update(self):
+        pass

File src/livemonitor/source.py

+
+class Source(object):
+    """A source retrieves data from somewhere and makes it accessible to compatible
+    measures.
+    """
+
+    def update(self):
+        pass

File src/livemonitor/static/js/monitor.js

-var charts = {};
+var series_index = {};
 
 var chart_options = {
     minValue: 0,
     fillStyle: 'rgba(0, 255, 0, 0.4)',
     lineWidth: 3};
 
-
-function setup_metric(i, name) {
-    charts[name] = {};
-    metric = charts[name];
-    metric.name = name;
-    metric.chart = new SmoothieChart(chart_options);
-    metric.chart.streamTo(document.getElementById(name), 2000);
-    metric.timeseries = new TimeSeries();
-    metric.chart.addTimeSeries(
-        metric.timeseries, timeseries_options);
-};
+// Picked palette from http://www.colourlovers.com/palette/160924/DANCE_TO_THE_CHARTS
+var strokeStyles = ["rgb(183,247,49)",
+                    "rgb(255,194,38)",
+                    "rgb(255,41,126)",
+                    "rgb(194,61,255)",
+                    "rgb(69,224,211)"];
+var fillStyles = ["rgba(183,247,49,0.2)",
+                  "rgba(255,194,38,0.2)",
+                  "rgba(255,41,126,0.2)",
+                  "rgba(194,61,255,0.2)",
+                  "rgba(69,224,211,0.2)"];
 
 
 function update_metrics(message) {
     data = $.parseJSON(message.data);
     $.each(data, function(metric, measure) {
-        charts[metric].timeseries.append(measure.time, measure.value); });
+        var timeseries = series_index[metric];
+        timeseries.append(measure.time, measure.value); });
 };
 
 
-$(document).ready(function(){
-    var metrics = ['haproxy_requests', 'haproxy_errors'];
-    $.each(metrics, setup_metric);
+function setup_charts(charts_data) {
+    var template = $('#chart_template');
+    var container = $('#charts');
 
-    charts['haproxy_errors'].timeseries.referenceSeries = charts['haproxy_requests'].timeseries;
+    $.each(charts_data, function(i, names) {
+        var chart_obj = {};
+        // Setup DOM
+        var chart_dom = template.clone();
+        chart_dom.removeAttr('style');
+        $('h3', chart_dom).text(names[0]);
+        container.append(chart_dom);
 
+        // Setup chart
+        var chart = new SmoothieChart(chart_options);
+        chart.streamTo($('canvas', chart_dom).get(0));
+
+        $.each(names, function(i, name) {
+            var timeseries = new TimeSeries();
+            var options = $.extend({}, timeseries_options);
+            options.strokeStyle = strokeStyles[i % strokeStyles.length];
+            options.fillStyle = fillStyles[i % fillStyles.length];
+            chart.addTimeSeries(timeseries, options);
+            chart_obj.timeseries = timeseries;
+            series_index[name] = timeseries;
+        });
+    });
+
+    init_websocket();
+};
+
+function init_websocket() {
     var ws = new WebSocket("ws://" + document.domain + ":5000/data");
     ws.onmessage = update_metrics;
+};
+
+$(document).ready(function(){
+    $.ajax('/charts', {dataType: 'json', success: setup_charts});
 });

File src/livemonitor/static/js/smoothie.js

 function TimeSeries(options) {
   options = options || {};
   options.resetBoundsInterval = options.resetBoundsInterval || 3000; // Reset the max/min bounds after this many milliseconds
-  options.resetBounds = options.resetBounds || true; // Enable or disable the resetBounds timer
+  if (!options.hasOwnProperty('resetBounds')) {
+      options.resetBounds = true;
+  }
   this.options = options;
   this.data = [];
   this.referenceSeries = options.referenceSeries || this; 

File src/livemonitor/templates/index.html

       <div class="navbar-inner">
         <div class="container">
           <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
-            <span class="icon-bar"></span>
+            <span class="icon-bar">asdf</span>
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
           </a>
       </div>
     </div>
 
-    <div class="container">
+    <div class="container" id="charts">
+    </div> <!-- /container -->
 
-        <div class="row">
-            <div class="span12">
-            <h3>HAProxy</h3>
-            </div>
-            <div class="span6">
-              <h3>Requests</h3>
-              <canvas id="haproxy_requests" width="500" height="100"></canvas>
-            </div>
-            <div class="span6">
-              <h3>Errors</h3>
-              <canvas id="haproxy_errors" width="500" height="100"></canvas>
-            </div>
+    <div class="row" id="chart_template" style="display:none">
+        <div class="span6">
+        <h3>Title</h3>
+        <canvas id="haproxy_requests" width="500" height="100"></canvas>
         </div>
-
-    </div> <!-- /container -->
+    </div>
 
     <!-- Le javascript
     ================================================== -->