Commits

Chris Mutel committed b7b50c0

Initial checkin

Comments (0)

Files changed (10)

+Copyright (c) 2011, ETH Zürich
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+include *.txt
+include README
+recursive-include mpwebstatus *.py
+A web status dashboard for Python's Multiprocessing
+***************************************************
+
+The `multiprocessing <http://docs.python.org/2/library/multiprocessing.html>`_ is a great way to distribute jobs to multiple cores, but has no simple way to indicate the status of each job. This package provides a callback function for multiprocessing jobs, a daemon to listen to status updates, and a a simple web dashboard to monitor jobs.

mpwebstatus/__init__.py

+from webapp import app
+from callback import simple_callback, curried_callback

mpwebstatus/bin/mp_web_controller.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Multiprocessing web status dashboard.
+
+Usage:
+  mp_web_controller.py [--port=<port>] [--nobrowser] [--insecure]
+  mp_web_controller.py -h | --help
+  mp_web_controller.py --version
+
+Options:
+  -h --help     Show this screen.
+  --version     Show version.
+  --nobrowser   Don't automatically open a browser tab.
+  --insecure    Allow outside connections.
+
+"""
+from mpwebstatus import app
+from docopt import docopt
+import threading
+import webbrowser
+
+
+if __name__ == "__main__":
+    args = docopt(__doc__, version='Multiprocessing web status 0.1')
+    port = int(args.get("--port", False) or 5000)
+    host = "0.0.0.0" if args.get("--insecure", False) else "localhost"
+
+    if not args["--nobrowser"]:
+        url = "http://127.0.0.1:{}/".format(port)
+        threading.Timer(1., lambda: webbrowser.open_new_tab(url)).start()
+
+    app.run(host=host, port=port)

mpwebstatus/callback.py

+# -*- coding: utf-8 -*-
+import requests
+import json
+
+
+def simple_callback(address, **data):
+    """Simple callback function that uploads job-specific status data to a given address."""
+    try:
+        requests.post(address, data=json.dumps(data))
+    except:
+        pass
+
+
+class curried_callback(object):
+    def __init__(self, url):
+        self.url = url
+        self.failed = 0
+        self.available = True
+
+    def __call__(self, **kwargs):
+        if not self.available:
+            return
+        try:
+            requests.post(self.url, data=json.dumps(kwargs))
+        except:
+            # Error posting to collection daemon
+            self.failed += 1
+            if self.failed > 5:
+                self.available = False

mpwebstatus/templates/index.html

+<!doctype html>
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+    <title>Multiprocessing dashboard</title>
+    <meta name="description" content="A web status dashboard for multiprocessing tasks.">
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.8.3/jquery.min.js"></script>
+    <style type="text/css">
+    table {
+        text-align: left;
+        border-collapse: collapse;
+        margin: 25px;
+    }
+    td {
+        border-right: 30px solid white;
+        border-left: 30px solid white;
+        padding: 12px 2px 0
+    }
+    th {
+        font-weight: normal;
+        font-size: 18px;
+        border-bottom: 2px solid midnightblue;
+        border-right: 30px solid white;
+        border-left: 30px solid white;
+        color: royalblue;
+        padding: 8px 2px
+    }
+    </style>
+</head>
+<body>
+    <h1>Multiprocessing dashboard</h1>
+    <hr>
+    <table id="jobs"></table>
+<script type="text/javascript">
+var get_status = function(j) {
+    if (j.finished === true) {
+        return "Finished"
+    } else {
+        return "Working"
+    };
+};
+
+var n = function (x) {
+    return parseFloat(x).toFixed(2)
+};
+
+function update() {    
+    $.get('/status', function(data){
+        var json = jQuery.parseJSON(data);
+        $(function () {
+            var content = '<thead><tr><th>Task</th><th>Status</th><th>ETA (seconds)</th><th>Elapsed (seconds)</th><th>Progress</th><th>Total</th></tr></thead>';
+            //content += '<tbody>'; -- **superfluous**
+            for (var i = 0; i < json.length; i++) {
+                content += '<tr><td>' + json[i].task + '</td><td>'
+                content += get_status(json[i]) + '</td><td>' + n(json[i].eta)
+                content += '</td><td>' + n(json[i].elapsed) + '</td><td>'
+                content += json[i].progress + '</td><td>' + json[i].total
+                content += '</td></tr>'
+            }
+             $('#jobs').html(content);
+       });  
+    });
+};
+
+setInterval(update, 1000);
+</script>
+</body>
+</html>

mpwebstatus/webapp.py

+# -*- coding: utf-8 -*-
+from flask import Flask, request, render_template
+import json
+import time
+
+app = Flask(__name__)
+
+global datastore
+datastore = {}
+
+
+@app.route("/")
+def index():
+    return render_template("index.html")
+
+
+@app.route("/update/<task>", methods=["POST"])
+def update(task):
+    data = json.loads(request.data)
+    if task not in datastore:
+        data["_born"] = data["_ping"] = time.time()
+    else:
+        data["_ping"] = time.time()
+        data["_born"] = datastore[task]["_born"]
+    datastore[task] = data
+    return ""
+
+
+@app.route("/status")
+def status():
+    print datastore
+    d = [{
+        "task": k,
+        "eta": v.get("eta", 0),
+        "elapsed": time.time() - v.get("_born", 0),
+        "ping": v.get("_ping", 0),
+        "finished": v.get("finished", False),
+        "progress": v.get("progress", 0),
+        "total": v.get("total", 0),
+        } for k, v in datastore.iteritems()]
+    d.sort(key=lambda x: x["task"])
+    return json.dumps(d)
+flask
+requirements
+docopt
+from distutils.core import setup
+
+setup(
+    name='mpwebstatus',
+    version='0.1',
+    author='Chris Mutel',
+    author_email='cmutel@gmail.com',
+    url='https://bitbucket.org/cmutel/mpwebstatus',
+    packages=['mpwebstatus'],
+    package_data={'mpwebstatus': ["templates/*.html"]},
+    scripts=["mpwebstatus/bin/mp_web_controller.py"],
+    license='BSD 2-clause; LICENSE.txt',
+    requires=["docopt", "requests", "flask"],
+    long_description=open('README').read(),
+)