Commits

Peter Ward committed 22c338b

Start of generic chart views.

  • Participants
  • Parent commits acff04b

Comments (0)

Files changed (7)

 from collections import defaultdict
-from glob import glob
 from itertools import izip as zip
 import json
-from os.path import join, split
+import os
+from os import path
+import re
 from subprocess import Popen, PIPE
 
 from flask import (
     Flask,
-    abort, current_app, jsonify, request, render_template,
+    abort, current_app, g, jsonify, request, render_template,
 )
 
-from utils import cache_time
-
-from graphdefs import GRAPH_DEFINITIONS
+#from utils import cache_time
 
 app = Flask(__name__)
 app.config['DATA_DIR'] = 'data'
+app.config['GRAPH_DEFINITIONS'] = 'definitions.json'
 
-app.config['GRAPH_DEFINITIONS'] = GRAPH_DEFINITIONS
+def get_host_files(host_dir):
+    """get all available files for host"""
+    for dirpath, _, filenames in os.walk(host_dir):
+        for filename in filenames:
+            yield path.relpath(path.join(dirpath, filename), host_dir)
 
-def rdefaultdict():
-    return defaultdict(rdefaultdict)
+def serialise_dict(d):
+    return tuple(sorted(d.items()))
 
-def rdict(d):
-    n = {}
-    for key, value in d.iteritems():
-        if isinstance(value, defaultdict):
-            value = rdict(value)
-        n[key] = value
-    return n
+def substitute(target, resolution):
+    for k, v in resolution.iteritems():
+        target = target.replace('<' + k + '>', v)
+    return target
 
-@cache_time(60)
-def get_all_datasets():
+def resolve_variables(host_files, file_specs):
+    resolutions = None
+
+    for file_spec in file_specs:
+        regex = re.compile(
+            re.escape(file_spec)
+            .replace(r'\<', '(?P<')
+            .replace(r'\>', '>[^/]*)')
+        )
+
+        these_resolutions = set()
+
+        matches = []
+        for filename in host_files:
+            m = regex.match(filename)
+            if m is not None:
+                these_resolutions.add(serialise_dict(m.groupdict()))
+
+        if resolutions is None:
+            resolutions = these_resolutions
+        else:
+            resolutions.intersection_update(these_resolutions)
+
+        if not resolutions:
+            break
+
+    resolutions = [
+        dict(r)
+        for r in resolutions
+    ]
+
+    assert all(
+        set(r.keys()) == set(resolutions[0].keys())
+        for r in resolutions
+    )
+
+    return resolutions
+
+#@cache_time(60)
+def get_views():
+    if hasattr(g, 'views'):
+        return g.views
+
+    with open(current_app.config['GRAPH_DEFINITIONS'], 'rU') as fp:
+        definitions = json.load(fp)
+
+    graph_defs = definitions['graphs']
+    type_defs = definitions['types']
+
     data_dir = current_app.config['DATA_DIR']
-    datasets = rdefaultdict()
 
-    for filename in glob(join(data_dir, '*', '*', '*.rrd')):
-        part, dataset = split(filename)
-        dataset, _ = dataset.rsplit('.rrd', 1)
-        part, series = split(part)
-        part, host = split(part)
-        assert part == data_dir, 'might be slashes'
-        datasets[host][series][dataset] = filename
+    views = {}
 
-    return rdict(datasets)
+    for hostname in os.listdir(data_dir):
+        host_dir = path.join(data_dir, hostname)
+
+        host_files = list(get_host_files(host_dir))
+
+        host_views = defaultdict(list)
+
+        for name, specs in graph_defs.iteritems():
+            for spec in specs:
+                spec['type_def'] = type_def = type_defs[spec['type']]
+
+                file_specs = spec['files'].values()
+
+                spec_graphs = defaultdict(list)
+
+                for resolution in resolve_variables(host_files, file_specs):
+                    resolved_name = substitute(name, resolution)
+                    spec_graphs[resolved_name].append(resolution)
+
+                # take each view generated by this spec, and append the graph
+                for k, v in spec_graphs.iteritems():
+                    host_views[k].append((spec, v))
+
+        views[hostname] = dict(host_views)
+
+    g.views = views
+    return views
+
+app.jinja_env.globals['get_views'] = get_views
 
 def make_time(n):
     if n == 0:
         return 'end' + str(n)
     return str(n)
 
-def get_rrd_data(filename, ds_type, start=-60*60, end=0, step=None):
-    definition = current_app.config['GRAPH_DEFINITIONS'][ds_type]
+def get_rrd_data(hostname, graph, start=-60*60, end=0, step=None):
+    data_dir = current_app.config['DATA_DIR']
+    host_dir = path.join(data_dir, hostname)
+
+    spec, resolutions = graph
+
+    graph_args = []
+    for resolution in resolutions:
+        variables = dict(resolution)
+
+        for key, filename in spec['files'].iteritems():
+            variables[key] = path.join(
+                host_dir,
+                substitute(filename, resolution),
+            )
+
+        for line in spec['type_def']:
+            graph_args.append(substitute(line, variables))
 
     if step is None:
         step = int(128 / (end - start))
         '--end', end,
         '--step', step,
     ]
-    for arg in definition:
-        args.append(
-            arg.replace('{file}', filename)
-        )
+    args.extend(graph_args)
 
     p = Popen(
         args=args,
 
 @app.route('/')
 def show_all():
-    all_datasets = get_all_datasets()
-    return render_template(
-        'index.html',
-        all_datasets=all_datasets,
-    )
+    return render_template('index.html')
 
 @app.route('/<hostname>')
 def show_host(hostname):
-    all_datasets = get_all_datasets()
-    if hostname not in all_datasets:
+    return show_view(hostname, '')
+
+@app.route('/<hostname>/<path:view>')
+def show_view(hostname, view):
+    views = get_views()
+    if hostname not in views:
         abort(404)
-    host = all_datasets[hostname]
+
+    host = views[hostname]
+    if view not in host:
+        abort(404)
+
+    charts = host[view]
 
     return render_template(
-        'host.html',
-        all_datasets=all_datasets,
-        hostname=hostname,
-        host=host,
+        'view.html',
+        current_hostname=hostname,
+        current_view=view,
+        charts=charts,
     )
 
-@app.route('/<hostname>/<series>')
-def show_series(hostname, series):
-    all_datasets = get_all_datasets()
-    if hostname not in all_datasets:
-        abort(404)
-    host = all_datasets[hostname]
-    if series not in host:
+@app.route('/<hostname>/<path:view>/<int:n>.json')
+def get_data(hostname, view, n):
+    views = get_views()
+    if hostname not in views:
         abort(404)
 
-    datasets = {}
-    for name, filename in host[series].items():
-        ds_type = name.split('-', 1)[0]
-        datasets[name] = get_rrd_data(filename, ds_type)
+    host = views[hostname]
+    if view not in host:
+        abort(404)
 
-    return render_template(
-        'series.html',
-        all_datasets=all_datasets,
-        hostname=hostname,
-        series=series,
-        datasets=datasets,
-    )
+    graphs = host[view]
+    if not (0 <= n < len(graphs)):
+        abort(404)
 
-@app.route('/<hostname>/<series>/<dataset>.json')
-def get_dataset(hostname, series, dataset):
-    all_datasets = get_all_datasets()
-    if hostname not in all_datasets:
-        abort(404)
-    host = all_datasets[hostname]
-    if series not in host:
-        abort(404)
-    datasets = host[series]
-    if dataset not in datasets:
-        abort(404)
-    filename = datasets[dataset]
+    graph = graphs[n]
 
     step = None
     if 'step' in request.args:
     end = request.args.get('end', 0)
     end = int(end)
 
-    ds_type = dataset.split('-', 1)[0]
     return jsonify(
         get_rrd_data(
-            filename, ds_type,
+            hostname, graph,
             start, end, step,
         )
     )
 
 if __name__ == '__main__':
-    app.run(debug=True)
+    app.run(debug=True, host='0.0.0.0', port=5001)

File definitions.json

+{
+    "types": {
+        "load": [
+            "DEF:shortterm=<file>:shortterm:AVERAGE",
+            "DEF:midterm=<file>:midterm:AVERAGE",
+            "DEF:longterm=<file>:longterm:AVERAGE",
+            "XPORT:shortterm:shortterm (1m)",
+            "XPORT:midterm:midterm (3m)",
+            "XPORT:longterm:longterm (5m)"
+        ],
+
+        "cpu-summary": [
+            "DEF:idle-<num>=<idle>:value:AVERAGE",
+            "DEF:nice-<num>=<nice>:value:AVERAGE",
+            "DEF:user-<num>=<user>:value:AVERAGE",
+            "DEF:wait-<num>=<wait>:value:AVERAGE",
+            "DEF:system-<num>=<system>:value:AVERAGE",
+            "DEF:softirq-<num>=<softirq>:value:AVERAGE",
+            "DEF:interrupt-<num>=<interrupt>:value:AVERAGE",
+            "DEF:steal-<num>=<steal>:value:AVERAGE",
+
+            "CDEF:cpu-<num>=1,idle-<num>,idle-<num>,nice-<num>,user-<num>,wait-<num>,system-<num>,softirq-<num>,interrupt-<num>,steal-<num>,+,+,+,+,+,+,+,/,-,100,*",
+            "XPORT:cpu-<num>:cpu-<num>"
+        ],
+
+        "cpu": [
+            "DEF:idle=<idle>:value:AVERAGE",
+            "DEF:nice=<nice>:value:AVERAGE",
+            "DEF:user=<user>:value:AVERAGE",
+            "DEF:wait=<wait>:value:AVERAGE",
+            "DEF:system=<system>:value:AVERAGE",
+            "DEF:softirq=<softirq>:value:AVERAGE",
+            "DEF:interrupt=<interrupt>:value:AVERAGE",
+            "DEF:steal=<steal>:value:AVERAGE",
+            "XPORT:idle:idle",
+            "XPORT:nice:nice",
+            "XPORT:user:user",
+            "XPORT:wait:wait",
+            "XPORT:system:system",
+            "XPORT:softirq:softirq",
+            "XPORT:steal:steal"
+        ],
+
+        "default": [
+            "DEF:value=<file>:<value>:AVERAGE",
+            "XPORT:value:<label>"
+        ]
+    },
+    "graphs": {
+        "cpu": [{
+            "graph-type": "line",
+            "type": "cpu-summary",
+            "files": {
+                "idle": "cpu-<num>/cpu-idle.rrd",
+                "nice": "cpu-<num>/cpu-nice.rrd",
+                "user": "cpu-<num>/cpu-user.rrd",
+                "wait": "cpu-<num>/cpu-wait.rrd",
+                "system": "cpu-<num>/cpu-system.rrd",
+                "softirq": "cpu-<num>/cpu-softirq.rrd",
+                "interrupt": "cpu-<num>/cpu-interrupt.rrd",
+                "steal": "cpu-<num>/cpu-steal.rrd"
+            }
+        }],
+
+        "load": [{
+            "graph-type": "line",
+            "type": "load",
+            "files": {"file": "load/load.rrd"}
+        }],
+
+        "cpu/<num>": [{
+            "graph-type": "stacked",
+            "type": "cpu",
+            "files": {
+                "idle": "cpu-<num>/cpu-idle.rrd",
+                "nice": "cpu-<num>/cpu-nice.rrd",
+                "user": "cpu-<num>/cpu-user.rrd",
+                "wait": "cpu-<num>/cpu-wait.rrd",
+                "system": "cpu-<num>/cpu-system.rrd",
+                "softirq": "cpu-<num>/cpu-softirq.rrd",
+                "interrupt": "cpu-<num>/cpu-interrupt.rrd",
+                "steal": "cpu-<num>/cpu-steal.rrd"
+            }
+        }]
+    }
+}

File static/js/charts.js

+Highcharts.setOptions({
+    global: {useUTC: false}
+});
+
+function add_chart(id, title, url, series, n) {
+    var chart = new Highcharts.Chart({
+        chart: {
+            renderTo: id,
+            defaultSeriesType: 'spline'
+        },
+        title: {text: title},
+        xAxis: {
+            type: 'datetime',
+            tickPixelInterval: 150
+        },
+        yAxis: {title: ''},
+        tooltip: {
+            formatter: function() {
+                    return '<b>'+ this.series.name +'</b><br/>'+
+                    Highcharts.dateFormat('%Y-%m-%d %H:%M:%S', this.x) +'<br/>'+
+                    Highcharts.numberFormat(this.y, 2);
+            }
+        },
+        legend: {enabled: true},
+        series: series
+    });
+
+    function refresh_data() {
+        $.ajax({
+            url: url,
+            success: function(data) {
+                for (var name in data) {
+                    var series = null;
+                    for (var i = 0; i < chart.series.length; i++) {
+                        if (chart.series[i].name == name) {
+                            series = chart.series[i];
+                            break;
+                        }
+                    }
+
+                    var series_data = data[name];
+
+                    if (series === null) {
+                        chart.addSeries({
+                            name: name,
+                            data: series_data
+                        });
+                    } else if (series.data.length == 0) {
+                        series.setData(series_data, false);
+                    } else {
+                        var last_timestamp = series.data[series.data.length - 1].x;
+                        for (var i = 0; i < series_data.length; i++) {
+                            var point = series_data[i];
+                            if (point[0] <= last_timestamp)
+                                continue;
+                            series.addPoint(point, false, true);
+                        }
+                    }
+                }
+                chart.redraw();
+                setTimeout(refresh_data, 5000);
+            },
+            cache: false
+        });
+    }
+
+    refresh_data();
+}

File templates/base.html

                 url_for('.static', filename='js/jquery-1.7.1.min.js')
             }}">\x3C/script>')</script>
         <script src="{{ url_for('.static', filename='js/highcharts/highcharts.js') }}"></script>
-        <script type="text/javascript">
-{% block javascript %}
-{% endblock %}
-        </script>
+        <script src="{{ url_for('.static', filename='js/charts.js') }}"></script>
     </head>
     <body>
         <div class="main">
             <div class="sidebar">
-            {% block sidebar %}
+            {%- block sidebar %}
                 <ul class="hosts">
-                    {% for hn in all_datasets |sort %}
-                    {% if hostname == hn %}
+                    {%- for hostname, host_views in get_views().items() |sort %}
+                    {%- if hostname == current_hostname %}
                     <li class="expanded">
-                    {% else %}
+                    {%- else %}
                     <li class="collapsed">
-                    {% endif %}
-                    <a href="{{ url_for('show_host', hostname=hn) }}">{{ hn }}</a>
+                    {%- endif -%}
+                    {#<a href="{{
+                        url_for('show_host', hostname=hostname)
+                    }}">#}{{ hostname }}{#</a>#}
                     <ul class="series">
-                    {% for series in all_datasets[hn] |sort %}
-                        <li>
-                        <a href="{{
-                            url_for('show_series',
-                                hostname=hn,
-                                series=series
+                    {%- for view in host_views |sort %}
+                        <li><a href="{{
+                            url_for('show_view',
+                                hostname=hostname,
+                                view=view
                             )
-                        }}">{{series}}</a>
-                        </li>
-                    {% endfor %}
+                        }}">{{view}}</a></li>
+                    {%- endfor %}
                     </ul>
                     </li>
-                    {% endfor %}
+                    {%- endfor %}
                 </ul>
-            {% endblock %}
+            {%- endblock %}
             </div>
             <div class="content">
             {% block content %}

File templates/host.html

-{% extends "base.html" %}
-
-{% block content %}
-<h1>{{ hostname }}</h1>
-<p>Choose a host on the left to view more detail
-<p>TODO: put summary charts here
-{% endblock %}
-

File templates/series.html

-{% extends "base.html" %}
-
-{% block javascript %}
-Highcharts.setOptions({
-    global: {useUTC: false}
-});
-
-function add_chart(id, title, url, series, n) {
-    var chart = new Highcharts.Chart({
-        chart: {
-            renderTo: id,
-            defaultSeriesType: 'spline'
-        },
-        title: {text: title},
-        xAxis: {
-            type: 'datetime',
-            tickPixelInterval: 150
-        },
-        yAxis: {title: ''},
-        tooltip: {
-            formatter: function() {
-                    return '<b>'+ this.series.name +'</b><br/>'+
-                    Highcharts.dateFormat('%Y-%m-%d %H:%M:%S', this.x) +'<br/>'+
-                    Highcharts.numberFormat(this.y, 2);
-            }
-        },
-        legend: {enabled: true},
-        series: series
-    });
-
-    var series_lookup = {};
-    for (var i = 0; i < series.length; i++) {
-        series_lookup[series[i]['name']] = i;
-    }
-
-    function refresh_data() {
-        $.ajax({
-            url: url,
-            success: function(data) {
-                for (var name in data) {
-                    var new_data = data[name];
-                    var series = chart.series[series_lookup[name]];
-                    if (series.data.length == 0)
-                        series.setData(new_data, false);
-                    else {
-                        var last_timestamp = series.data[series.data.length - 1].x;
-                        for (var i = 0; i < new_data.length; i++) {
-                            var point = new_data[i];
-                            if (point[0] <= last_timestamp)
-                                continue;
-                            series.addPoint(point, false, true);
-                        }
-                    }
-                }
-                chart.redraw();
-                setTimeout(refresh_data, 5000);
-            },
-            cache: false
-        });
-    }
-
-    refresh_data();
-}
-{% endblock %}
-
-{% block content %}
-<h1>{{ hostname }}: {{ series }}</h1>
-
-{% for ds in datasets %}
-<div id="chart-{{ ds }}" class="chart"></div>
-{% endfor %}
-
-<script type="text/javascript">
-{% for ds, gseries in datasets.items() %}
-    add_chart(
-        "chart-{{ ds }}",
-        "{{ ds }}",
-        "{{
-            url_for('get_dataset',
-                hostname=hostname,
-                series=series,
-                dataset=ds,
-                start=-60*10,
-            )
-        }}",
-        [
-        {%- for name, data in gseries.items() %}
-            {name: '{{ name }}', data: {# data |tojson #}[]}
-            {%- if not loop.last %},{% endif %}
-        {%- endfor %}
-        ],
-        20
-    );
-{% endfor %}
-</script>
-{% endblock %}
-

File templates/view.html

+{% extends "base.html" %}
+
+{% block content %}
+<h1>{{ current_hostname }}: {{ current_view }}</h1>
+
+{% for chart in charts %}
+<div id="chart-{{ loop.index0 }}" class="chart"></div>
+{% endfor %}
+
+<script type="text/javascript">
+{% for chart in charts %}
+    add_chart(
+        "chart-{{ loop.index0 }}",
+        "{{ chart.type }}",
+        "{{
+            url_for('get_data',
+                hostname=current_hostname,
+                view=current_view,
+                n=loop.index0,
+                start=-60*10,
+            )
+        }}",
+        [],
+        20
+    );
+{% endfor %}
+</script>
+{% endblock %}
+