Commits

Ian Bicking committed 8c182a1

Add parser for silver error log format. Add a checkpoint idea, where you can view just recent logs

Comments (0)

Files changed (6)

 [production]
 app_name = silverlog
 runner = src/silverlog/silver-runner.py
+service.files =
 
 # Should there be a setting of some sort to mark apps that work over
 # the entire system?

silverlog/__init__.py

 import json
 import time
 import tempita
+import cPickle as pickle
 from webob.dec import wsgify
 from webob import exc
 from webob import Response
     map.connect('list_logs', '/api/list-logs', method='list_logs')
     map.connect('log_view', '/api/log/{id}', method='log_view')
     map.connect('skipped_files', '/api/skipped-files', method='skipped_files')
+    map.connect('checkpoint', '/api/checkpoint', method='checkpoint')
 
     def __init__(self, dirs=None, template_base=None):
         if template_base:
         result = dict(
             path=log.path, group=log.group,
             id=log.id, description=log.description,
-            chunks=log.parse_chunks(),
+            chunks=log.parse_chunks(self.checkpoint_info(req)),
             log_type=log.parser)
         if 'nocontent' not in req.GET:
             result['content'] = log.content()
             skipped_files=self.log_set.skipped_files)
         return json_response(result)
 
+    def checkpoint(self, req):
+        if req.method == 'POST':
+            id = self.make_new_checkpoint()
+            return json_response(dict(checkpoint=id))
+        elif req.method == 'GET':
+            return json_response(dict(
+                checkpoints=[c for c in self.checkpoint_list()]))
+
+    def make_new_checkpoint(self):
+        date = time.gmtime()
+        timestamp = time.strftime('%Y%m%dT%H%M%S', date)
+        date = translate_date(date)
+        fn = self.checkpoint_filename(timestamp)
+        data = {}
+        for group in self.log_set.logs.values():
+            for log_filename in group.keys():
+                size = os.path.getsize(log_filename)
+                data[log_filename] = size
+        fp = open(fn, 'wb')
+        pickle.dump(data, fp)
+        fp.close()
+        return dict(id=timestamp, date=date)
+
+    def checkpoint_list(self):
+        result = []
+        for name in os.listdir(self.checkpoint_dir):
+            base, ext = os.path.splitext(name)
+            if ext != '.checkpoint':
+                continue
+            date = time.strptime(base, '%Y%m%dT%H%M%S')
+            result.append(dict(id=base, date=translate_date(date)))
+        return result
+
+    def checkpoint_info(self, req):
+        if not req.GET.get('checkpoint'):
+            return None
+        fn = self.checkpoint_filename(req.GET['checkpoint'])
+        fp = open(fn, 'rb')
+        data = pickle.load(fp)
+        fp.close()
+        return data
+
+    def checkpoint_filename(self, id):
+        assert re.search(r'^[a-zA-Z0-9]+$', id)
+        path = os.path.join(self.checkpoint_dir, id+'.checkpoint')
+        if not os.path.exists(os.path.dirname(path)):
+            os.makedirs(os.path.dirname(path))
+        return path
+
+    @property
+    def checkpoint_dir(self):
+        dir = os.path.join(os.environ['CONFIG_FILES'], 'checkpoints')
+        if not os.path.exists(dir):
+            os.makedirs(dir)
+        return dir
+
 NAMES = [
-    (r'^SILVER_DIR/apps/(?P<app>[^/]+)/error.log(?:\.(?P<number>\d+))?$',
+    (r'^SILVER_DIR/apps/(?P<app>[^/]+)/errors\.log(?:\.(?P<number>\d+))?$',
      ('{{app}}', '{{app}}: error log{{if number}} (backup {{number}}){{endif}}'),
      'silver_error_log'),
     (r'^SILVER_DIR/apps/(?P<app>[^/]+)/(?P<name>.*)(?:\.(?P<number>\d+))?$',
         fp.close()
         return c
 
-    def parse_chunks(self):
-        method = getattr(self, self.parser)
-        return method()
+    def parse_chunks(self, checkpoint_info):
+        place = None
+        if checkpoint_info and checkpoint_info.get(self.path):
+            place = checkpoint_info[self.path]
+        fp = open(self.path)
+        try:
+            if place:
+                print 'seeking %s to %s' % (self.path, place)
+                fp.seek(place)
+            else:
+                print 'No seek on %s' % self.path
+            method = getattr(self, self.parser)
+            result = method(fp)
+            #result['offset'] = place
+            return result
+        finally:
+            fp.close()
 
-    def generic_log(self):
-        fp = open(self.path)
+    def generic_log(self, fp):
         l = []
         for line in fp:
             l.append({'data': line.strip()})
         return l
 
-    def apache_access_log(self):
-        fp = open(self.path)
+    def apache_access_log(self, fp):
         l = []
         regex = re.compile(
             r'''
                 data = match.groupdict()
                 if data.get('app_name') == '-':
                     data['app_name'] = ''
-                data['date'] = self._translate_apache_date(data['date'])
+                data['date'] = translate_apache_date(data['date'])
             else:
                 data = {}
             data['data'] = line
             l.append(data)
         return l
 
-    @staticmethod
-    def _translate_apache_date(date):
-        return time.strftime('%B %d, %Y %H:%M:%S', time.strptime(date.split()[0], '%d/%b/%Y:%H:%M:%S'))
-
-    def apache_error_log(self):
-        fp = open(self.path)
+    def apache_error_log(self, fp):
         l = []
         regex = re.compile(
             r'''
             l.append(data)
         return l
 
-    def silver_setup_node_log(self):
-        fp = open(self.path)
+    def silver_setup_node_log(self, fp):
         l = []
         rerun = re.compile(
             r'''
             if match:
                 last_item.update(match.groupdict())
                 continue
-            if line == '-'*len(line):
+            if line == '-' * len(line):
                 last_item['data'] = '\n'.join(last_item['data'])
                 l.append(last_item)
                 last_item = {}
             l.append(last_item)
         return l
 
-    def apache_rewrite_log(self):
-        fp = open(self.path)
+    def apache_rewrite_log(self, fp):
         l = []
         regex = re.compile(
             r'''
                 last_item = l[-1]
             else:
                 data = match.groupdict()
-                data['date'] = self._translate_apache_date(data['date'])
+                data['date'] = translate_apache_date(data['date'])
                 if not data['message'].startswith('applying pattern'):
                     data['message'] = '  ' + data['message']
                 if (last_item
                     last_item = data
         return l
 
+    def silver_error_log(self, fp):
+        l = []
+        start_regex = re.compile(
+            r'''
+            Errors \s+ for \s+ request \s+
+            (?P<method>[A-Z]+) \s+
+            (?P<path>[^\s]+) \s+
+            \((?P<date>[^)]*)\):
+            ''', re.VERBOSE)
+        end_regex = re.compile(
+            r'''Finish errors for request''')
+        for line in fp:
+            line = line.rstrip()
+            match = start_regex.match(line)
+            if match:
+                l.append(match.groupdict())
+                try:
+                    l[-1]['date'] = translate_date(time.strptime(l[-1]['date'].split(',')[0], '%Y-%m-%d %H:%M:%S'))
+                except ValueError:
+                    # Just don't convert if it's weird
+                    pass
+                l[-1]['lines'] = []
+                continue
+            match = end_regex.match(line)
+            if match:
+                # Well, I guess we don't care
+                continue
+            if not l:
+                l.append(dict(method='unknown',
+                              path='unknown',
+                              date=None,
+                              lines=[]))
+            l[-1]['lines'].append(line)
+        for item in l:
+            item['message'] = '\n'.join(item.pop('lines'))
+        return l
+
     @property
     def id(self):
         id = self.path.replace('/', '_').strip('_')
     return Response(json.dumps(data),
                     content_type='application/json',
                     **kw)
+
+def translate_apache_date(date):
+    return translate_date(time.strptime(date.split()[0], '%d/%b/%Y:%H:%M:%S'))
+
+def translate_date(date):
+    return time.strftime('%B %d, %Y %H:%M:%S', date)

silverlog/static/index.html

 <script src="./sammy.js"></script>
 <script src="./pure.js"></script>
 <script src="./jquery.relatize_date.js"></script>
+<script src="./jquery.cookie.js"></script>
 <script src="./script.js"></script>
 <title>Silver Logs</title>
 </head>
 <body>
 
 <div id="header">
+  <span class="controls">
+    <select name="checkpoint" id="checkpoint">
+      <option value="">All items</option>
+    </select>
+  </span>
+
+
   <h1><a href="#/">Silver Logs:</a>
   <span id="title-slot"></span></h1>
+
 </div>
 
 <div id="body">
     </div>
   </div>
 
+  <div class="template-silver_error_log">
+    <div class="log-section">
+      <div class="log-section-header">
+        <span class="log-method">method</span>
+        <code class="log-path">path</code>
+        <span class="date log-date">date</span>
+      </div>
+      <pre class="log-message"></pre>
+    </div>
+  </div>
+
   <div class="template-apache_rewrite_log">
     <div class="log-section">
       <div class="log-section-header">

silverlog/static/jquery.cookie.js

+/**
+ * Cookie plugin
+ *
+ * Copyright (c) 2006 Klaus Hartl (stilbuero.de)
+ * Dual licensed under the MIT and GPL licenses:
+ * http://www.opensource.org/licenses/mit-license.php
+ * http://www.gnu.org/licenses/gpl.html
+ *
+ */
+
+/**
+ * Create a cookie with the given name and value and other optional parameters.
+ *
+ * @example $.cookie('the_cookie', 'the_value');
+ * @desc Set the value of a cookie.
+ * @example $.cookie('the_cookie', 'the_value', { expires: 7, path: '/', domain: 'jquery.com', secure: true });
+ * @desc Create a cookie with all available options.
+ * @example $.cookie('the_cookie', 'the_value');
+ * @desc Create a session cookie.
+ * @example $.cookie('the_cookie', null);
+ * @desc Delete a cookie by passing null as value. Keep in mind that you have to use the same path and domain
+ *       used when the cookie was set.
+ *
+ * @param String name The name of the cookie.
+ * @param String value The value of the cookie.
+ * @param Object options An object literal containing key/value pairs to provide optional cookie attributes.
+ * @option Number|Date expires Either an integer specifying the expiration date from now on in days or a Date object.
+ *                             If a negative value is specified (e.g. a date in the past), the cookie will be deleted.
+ *                             If set to null or omitted, the cookie will be a session cookie and will not be retained
+ *                             when the the browser exits.
+ * @option String path The value of the path atribute of the cookie (default: path of page that created the cookie).
+ * @option String domain The value of the domain attribute of the cookie (default: domain of page that created the cookie).
+ * @option Boolean secure If true, the secure attribute of the cookie will be set and the cookie transmission will
+ *                        require a secure protocol (like HTTPS).
+ * @type undefined
+ *
+ * @name $.cookie
+ * @cat Plugins/Cookie
+ * @author Klaus Hartl/klaus.hartl@stilbuero.de
+ */
+
+/**
+ * Get the value of a cookie with the given name.
+ *
+ * @example $.cookie('the_cookie');
+ * @desc Get the value of a cookie.
+ *
+ * @param String name The name of the cookie.
+ * @return The value of the cookie.
+ * @type String
+ *
+ * @name $.cookie
+ * @cat Plugins/Cookie
+ * @author Klaus Hartl/klaus.hartl@stilbuero.de
+ */
+jQuery.cookie = function(name, value, options) {
+    if (typeof value != 'undefined') { // name and value given, set cookie
+        options = options || {};
+        if (value === null) {
+            value = '';
+            options.expires = -1;
+        }
+        var expires = '';
+        if (options.expires && (typeof options.expires == 'number' || options.expires.toUTCString)) {
+            var date;
+            if (typeof options.expires == 'number') {
+                date = new Date();
+                date.setTime(date.getTime() + (options.expires * 24 * 60 * 60 * 1000));
+            } else {
+                date = options.expires;
+            }
+            expires = '; expires=' + date.toUTCString(); // use expires attribute, max-age is not supported by IE
+        }
+        // CAUTION: Needed to parenthesize options.path and options.domain
+        // in the following expressions, otherwise they evaluate to undefined
+        // in the packed version for some reason...
+        var path = options.path ? '; path=' + (options.path) : '';
+        var domain = options.domain ? '; domain=' + (options.domain) : '';
+        var secure = options.secure ? '; secure' : '';
+        document.cookie = [name, '=', encodeURIComponent(value), expires, path, domain, secure].join('');
+    } else { // only name given, get cookie
+        var cookieValue = null;
+        if (document.cookie && document.cookie != '') {
+            var cookies = document.cookie.split(';');
+            for (var i = 0; i < cookies.length; i++) {
+                var cookie = jQuery.trim(cookies[i]);
+                // Does this cookie string begin with the name we want?
+                if (cookie.substring(0, name.length + 1) == (name + '=')) {
+                    cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                    break;
+                }
+            }
+        }
+        return cookieValue;
+    }
+};

silverlog/static/script.js

   return $.ajax(options);
 };
 
+var currentCheckpoint = null;
+
+$(function () {
+
+  $.ajax({
+    url: "./api/checkpoint",
+    dataType: "json",
+    success: function (result) {
+      $.each(result.checkpoints, function () {
+        var el = $('<option></option>').attr('value', this.id).attr('title', this.date).text($.relatizeDate.timeAgoInWords(new Date(this.date), true));
+        $('#checkpoint').append(el);
+      });
+      if ($.cookie('checkpoint')) {
+        $('#checkpoint option[value="'+$.cookie('checkpoint')+'"]').select();
+      }
+      $('#checkpoint').append(
+        $('<option id="checkpoint-add-new" value="__new__">New checkpoint</option>'));
+      $('#checkpoint').bind('change', function () {
+        currentCheckpoint = $('#checkpoint').val() || null;
+        $.cookie('checkpoint', currentCheckpoint || '');
+        if (currentCheckpoint == "__new__") {
+          $.ajax({
+            url: "./api/checkpoint",
+            dataType: "json",
+            type: "POST",
+            success: function (result) {
+              var el = $('<option></option>').attr('value', result.checkpoint.id).attr('title', result.checkpoint.date).text($.relatizeDate.timeAgoInWords(new Date(result.checkpoint.date), true));
+              $('#checkpoint-add-new').before(el);
+              el.select();
+              $.cookie('checkpoint', result.checkpoint.id);
+            }
+          });
+        }
+      });
+    }
+  });
+
+});
+
+function addCheckpoint(url) {
+  if (! currentCheckpoint) {
+    return url;
+  } else {
+    if (url.indexOf('?') == -1) {
+      url += '?';
+    } else {
+      url += '&';
+    }
+    url += 'checkpoint=' + currentCheckpoint;
+    return url;
+  }
+}
+
 var app = $.sammy(function () {
   this.element_selector = '#body';
 
     moveScreen('#log-view');
     $('#header #title-slot').text('Loading...');
     $.ajax({
-      url: "./api/log/" + this.params.log_id + "?nocontent",
+      url: addCheckpoint("./api/log/" + this.params.log_id + "?nocontent"),
       dataType: "json",
       success: function (result) {
         var log_type = result.log_type;
         ".log-message": "chunk.message"
       }
     }
+  },
+
+  "silver_error_log": {
+    "div.log-section": {
+      "chunk<-chunks": {
+        ".log-date": "chunk.date",
+        ".log-method": "chunk.method",
+        ".log-path": "chunk.path",
+        ".log-message": "chunk.message"
+      }
+    }
   }
 
 };
   el.show();
   return el;
 }
+
+function catcher(func) {
+  return function () {
+    try {
+      return func.apply(this, arguments);
+    } catch (e) {
+      Sammy.log('Got exception in function', func, e);
+      throw(e);
+    }
+  };
+}

silverlog/static/style.css

 .response-code-301 {
   color: #55a;
 }
+
+.controls {
+  float: right;
+}