Ian Bicking avatar Ian Bicking committed d6e209f

A bunch more formatting improvements, and see backup log files

Comments (0)

Files changed (4)

silverlog/__init__.py

     map = Mapper()
     map.connect('list_logs', '/api/list-logs', method='list_logs')
     map.connect('log_view', '/api/log/{id}', method='log_view')
+    map.connect('skipped_files', '/api/skipped-files', method='skipped_files')
 
     def __init__(self, dirs=None, template_base=None):
         if template_base:
     @wsgify
     def __call__(self, req):
         results = self.map.routematch(environ=req.environ)
-        print results, req.path_info
         if not results:
             return exc.HTTPNotFound()
         match, route = results
         return json_response(result)
 
     def log_view(self, req, id):
-        log = self.log_set.log_from_id(id)
+        try:
+            log = self.log_set.log_from_id(id)
+        except (OSError, IOError), e:
+            return json_response(
+                dict(error=str(e), log_id=id),
+                status='500')
         result = dict(
             path=log.path, group=log.group,
             id=log.id, description=log.description,
-            content=log.content(),
             chunks=log.parse_chunks(),
             log_type=log.parser)
+        if 'nocontent' not in req.GET:
+            result['content'] = log.content()
+        return json_response(result)
+
+    def skipped_files(self, req):
+        result = dict(
+            skipped_files=self.log_set.skipped_files)
         return json_response(result)
 
 NAMES = [
-    (r'^SILVER_DIR/apps/(?P<app>[^/]+)/error.log$',
-     ('{{app}}', '{{app}}: error log'),
+    (r'^SILVER_DIR/apps/(?P<app>[^/]+)/error.log(?:\.(?P<number>\d+))?$',
+     ('{{app}}', '{{app}}: error log{{if number}} (backup {{number}}){{endif}}'),
      'silver_error_log'),
-    (r'^SILVER_DIR/apps/(?P<app>[^/]+)/(?P<name>.*)$',
-     ('{{app}}', '{{app}}: {{name}}'),
+    (r'^SILVER_DIR/apps/(?P<app>[^/]+)/(?P<name>.*)(?:\.(?P<number>\d+))?$',
+     ('{{app}}', '{{app}}: {{name}}{{if number}} (backup {{number}}){{endif}}'),
      'generic_log'),
-    (r'^APACHE_DIR/access.log$',
-     ('system', 'Apache access log'),
+    (r'^APACHE_DIR/access.log(?:\.(?P<number>\d+))?$',
+     ('system', 'Apache access log{{if number}} (backup {{number}}){{endif}}'),
      'apache_access_log'),
-    (r'^APACHE_DIR/error.log$',
-     ('system', 'Apache error log'),
+    (r'^APACHE_DIR/error.log(?:\.(?P<number>\d+))?$',
+     ('system', 'Apache error log{{if number}} (backup {{number}}){{endif}}'),
      'apache_error_log'),
-    (r'^APACHE_DIR/rewrite.log$',
-     ('system', 'Apache rewrite log'),
+    (r'^APACHE_DIR/rewrite.log(?:\.(?P<number>\d+))?$',
+     ('system', 'Apache rewrite log{{if number}} (backup {{number}}){{endif}}'),
      'apache_rewrite_log'),
-    (r'^SILVER_DIR/setup-node.log',
-     ('system', 'silver setup-node log'),
+    (r'^SILVER_DIR/setup-node.log(?:\.(?P<number>\d+))?',
+     ('system', 'silver setup-node log{{if number}} (backup {{number}}){{endif}}'),
      'silver_setup_node_log'),
     ]
 
             l.append({'data': line.strip()})
         return l
 
-    silver_error_log = generic_log
-    apache_rewrite_log = generic_log
-
     def apache_access_log(self):
         fp = open(self.path)
         l = []
               \s+
             (?P<host>[^ ]+)
               \s+
-            (?P<app_name>[^ ]+)
+            "(?P<app_name>[^ ]+)"
               \s+
             (?P<milliseconds>\d+)
             )?
             match = regex.match(line)
             if match:
                 data = match.groupdict()
+                if data.get('app_name') == '-':
+                    data['app_name'] = ''
                 data['date'] = self._translate_apache_date(data['date'])
             else:
                 data = {}
             match = regex.match(line)
             if match:
                 data = match.groupdict()
-                if (last_data and data['date'] == last_data['date']
-                    and data['level'] == last_data['level']
-                    and data['remote_addr'] == last_data['remote_addr']):
+                if (last_data and data['date'] == last_data.get('date')
+                    and data['level'] == last_data.get('level')
+                    and data['remote_addr'] == last_data.get('remote_addr')):
                     last_data['data'] += '\n' + line
                     last_data['message'] += '\n' + data['message']
                     continue
+            elif last_data:
+                last_data['data'] += '\n' + line
+                last_data['message'] += '\n' + line
+                continue
             else:
                 data = {}
             data['data'] = line
             l.append(last_item)
         return l
 
+    def apache_rewrite_log(self):
+        fp = open(self.path)
+        l = []
+        regex = re.compile(
+            r'''
+            ^
+            (?P<remote_addr>[0-9\.]+)
+              \s+
+            - \s+
+            - \s+
+            \[(?P<date>[^\]]+)\]
+              \s+
+            \[(?P<request_id>[^\]]+)\]
+            \[(?P<request_id2>[^\]]+)\]
+              \s+
+            \((?P<level>\d+)\)
+              \s
+            (?P<message>.*)
+            ''', re.VERBOSE | re.I)
+        last_item = {}
+        for line in fp:
+            line = line.strip()
+            match = regex.match(line)
+            if not match:
+                l.append(dict(data=line))
+                last_item = l[-1]
+            else:
+                data = match.groupdict()
+                data['date'] = self._translate_apache_date(data['date'])
+                if not data['message'].startswith('applying pattern'):
+                    data['message'] = '  ' + data['message']
+                if (last_item
+                    and data['remote_addr'] == last_item['remote_addr']
+                    and data['date'] == last_item['date']
+                    and data['request_id'] == last_item['request_id']):
+                    last_item['data'] += '\n' + line
+                    last_item['message'] += '\n' + data['message']
+                else:
+                    data['data'] = line
+                    l.append(data)
+                    last_item = data
+        return l
+
     @property
     def id(self):
         id = self.path.replace('/', '_').strip('_')
         return id
 
-def json_response(data):
+def json_response(data, **kw):
     return Response(json.dumps(data),
-                    content_type='application/json')
+                    content_type='application/json',
+                    **kw)

silverlog/static/index.html

 <html>
 <head>
-<link href="/style.css" type="text/css" rel="stylesheet" />
-<!-- <script
+<link href="./style.css" type="text/css" rel="stylesheet" />
+<script
 src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
--->
+<!--
 <script
 src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.js"></script>
-<script src="/sammy.js"></script>
-<script src="/pure.js"></script>
-<script src="/jquery.relatize_date.js"></script>
-<script src="/script.js"></script>
+-->
+<script src="./sammy.js"></script>
+<script src="./pure.js"></script>
+<script src="./jquery.relatize_date.js"></script>
+<script src="./script.js"></script>
 <title>Silver Logs</title>
 </head>
 <body>
     <li class="logs"><a class="description link@href"
      href="">description</a></li>
   </ul>
+
+  <h3 id="skipped-list-title">Skipped files:</h3>
+  <ul id="skipped-list">
+    <li></li>
+  </ul>
 </div>
 
 <div id="log-view" style="display: none">
-  <h2 class="description">description</h2>
-  <pre class="content"></pre>
 </div>
 
 </div><!-- /#body -->
         <th>Time</th>
       </tr>
       <tr class="log-section">
-        <td class="date log-date"></td>
+        <td><span class="date log-date"></span></td>
         <td class="log-method"></td>
         <td><code class="log-path"></code></td>
         <td class="log-response_code"></td>
         <td class="log-user_agent"></td>
         <td><a href="" class="log-host"></a></td>
         <td class="log-app_name"></td>
-        <td class="log-milliseconds"></td>
+        <td class="log-time"></td>
       </tr>
     </table>
   </div>
     </div>
   </div>
 
+  <div class="template-apache_rewrite_log">
+    <div class="log-section">
+      <div class="log-section-header">
+        <span class="date log-date">date</span>
+        <code class="log-remote_addr">remote_addr</code>
+      </div>
+      <pre class="log-message"></pre>
+    </div>
+  </div>
+
 </div><!-- /templates -->
 
 </body>

silverlog/static/script.js

+$.fn.ajaxRender = function (options, declarations) {
+  var selector = this.selector;
+  if (typeof options == 'string') {
+    options = {url: options};
+  }
+  options.success = function (result) {
+    $(selector).render(result, declarations);
+  };
+  options.dataType = options.dataType || "json";
+  return $.ajax(options);
+};
+
 var app = $.sammy(function () {
   this.element_selector = '#body';
 
   this.get('#/', function (context) {
     moveScreen('#main');
     $('#main #log-list li:nth-child(1n+2)').remove();
+    $('#main #skipped-list li:nth-child(1n+2)').remove();
     $('#header #title-slot').text('index');
+
+    $('#main').ajaxRender(
+      "./api/list-logs",
+      {
+        "#log-list li": {
+          "log<-logs": {
+            "a": "log.description",
+            "a@href": function (ctx) {
+              return '#/view/'+ctx.item.id;
+            }
+          }
+        }
+      });
+
     $.ajax({
-      url: "/api/list-logs",
+      url: "./api/skipped-files",
       dataType: "json",
       success: function (result) {
         $('#main').render(
           result,
           {
-            "#log-list li": {
-              "log<-logs": {
-                "a": "log.description",
-                "a@href": function (ctx) {
-                  return '#/view/'+ctx.item.id;
-                }
+            "#skipped-list li": {
+              "skipped_file<-skipped_files": {
+                ".": "skipped_file"
               }
             }
           });
+        if (! result.skipped_files.length) {
+          $('#skipped-list').hide();
+          $('#skipped-list-title').hide();
+        } else {
+          $('#skipped-list').show();
+          $('#skipped-list-title').show();
+        }
       }
     });
+
   });
 
   this.get('#/view/:log_id', function (context) {
     moveScreen('#log-view');
+    $('#header #title-slot').text('Loading...');
     $.ajax({
-      url: "/api/log/"+this.params.log_id,
+      url: "./api/log/" + this.params.log_id + "?nocontent",
       dataType: "json",
       success: function (result) {
         var log_type = result.log_type;
         tmpl.show();
         tmpl.addClass('log-view');
         $('#log-view').append(tmpl);
-        console.log('rule', templateRules[log_type]);
-        console.log('data', result);
         tmpl = tmpl.render(
           result,
           templateRules[log_type]);
     "tr.log-section": {
       "chunk<-chunks": {
         ".log-date": "chunk.date",
+        ".log-date@title": "chunk.date",
         ".log-method": "chunk.method",
         ".log-path": "chunk.path",
         ".log-response_code": "chunk.response_code",
+        ".log-response_code@class+": function (ctx) {
+          return " response-code-"+ctx.item.response_code;
+        },
         ".log-response_bytes": "chunk.response_bytes",
-        ".log-referrer": function (ctx) {return ctx.item.referrer == "-" ? "" : ctx.item.referrer},
-        ".log-referrer@href": function (ctx) {return ctx.item.referrer == "-" ? "" : ctx.item.referrer},
+        ".log-referrer": function (ctx) {
+          var ref = ctx.item.referrer;
+          var host = ctx.item.host;
+          if (ref == "-")
+            return "";
+          if (ref.substr(7, host.length) == host) {
+            return ref.substr(host.length+7);
+          }
+          return ref.substr(ref.indexOf('://')+3);
+        },
+        ".log-referrer@href": function (ctx) {
+          return ctx.item.referrer == "-" ? "" : ctx.item.referrer;
+        },
         ".log-user_agent": "chunk.user_agent",
-        ".log-host": "chunk.host",
-        ".log-host@href": function (ctx) {return "http://"+ctx.item.host;},
+        ".log-host": function (ctx) {
+          var host = ctx.item.host;
+          if (host == "-") {
+            return "";
+          }
+          var idx = host.indexOf(".");
+          if (idx != -1) {
+              return host.substr(0, idx);
+          }
+          return host;
+        },
+        ".log-host@title": "chunk.host",
+        ".log-host@href": function (ctx) {
+          return ctx.item.host == "-" ? "" : "http://"+ctx.item.host;
+        },
         ".log-app_name": "chunk.app_name",
-        ".log-milliseconds": "chunk.milliseconds"
+        ".log-time": function (ctx) {return ctx.item.milliseconds/1000000;}
       }
     }
   },
     "div.log-section": {
       "chunk<-chunks": {
         ".log-warning-level": "chunk.level",
-        ".log-warning-level@class+": function (ctx) {return " log-warning-level-"+ctx.item.level},
+        ".log-warning-level@class+": function (ctx) {return " log-warning-level-"+ctx.item.level;},
+        ".log-date@title": "chunk.date",
         ".log-date": "chunk.date",
         ".log-client": "chunk.remote_addr",
         ".log-message": "chunk.message"
       }
     }
+  },
+
+  "apache_rewrite_log": {
+    "div.log-section": {
+      "chunk<-chunks": {
+        ".log-date": "chunk.date",
+        ".log-date@title": "chunk.date",
+        ".log-remote_addr": "chunk.remote_addr",
+        ".log-message": "chunk.message"
+      }
+    }
   }
 
 };

silverlog/static/style.css

   margin: 0;
 }
 
+.log-warning-level {
+  padding: 0 0.5em 0 0.5em;
+}
+
 .log-warning-level-notice {
   background-color: #ff9;
 }
   background-color: #f77;
 }
 
-.log-user_agent {
+.log-date, .log-method, .log-user_agent, .log-app_name {
   font-size: 70%;
 }
 
-.log-date {
-  font-size: 70%;
+.log-response_bytes, .log-time {
+  text-align: right;
 }
 
-.log-response_bytes {
-  text-align: right;
+.log-section-header th {
+  font-weight: normal;
+  border-bottom: 1px solid #000;
 }
+
+.response-code-200, .response-code-304 {
+  color: #999;
+}
+
+.response-code-404 {
+  color: #b72;
+}
+
+.response-code-500 {
+  color: #b00;
+}
+
+.response-code-301 {
+  color: #55a;
+}
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.