Commits

Virgil Dupras committed fdfd6aa

Added support for sitename in logfile entries (pretty cool).

Comments (0)

Files changed (10)

+from collections import namedtuple
 import yaml
 
 from .model.filter import FilterList
 
+LogFile = namedtuple('LogFile', 'path sitename')
+
 class Config:
     def __init__(self, filepath):
         self.filters = FilterList()
         self.logfiles = []
         conf = yaml.load(open(filepath, 'rt'))
         filters = conf.get('filters', [])
-        for filter_item in filters:
+        for filter_dict in filters:
             try:
-                type_ = filter_item['type']
-                value = filter_item['value']
+                type_ = filter_dict['type']
+                value = filter_dict['value']
             except KeyError:
                 continue
             self.filters.add_filter(type_, value)
-        self.logfiles = conf.get('logfiles', [])
+        self.logfiles = []
+        logfiles = conf.get('logfiles', [])
+        for logfile_dict in logfiles:
+            path = logfile_dict['path']
+            sitename = logfile_dict.get('sitename')
+            self.logfiles.append(LogFile(path, sitename))
     

core/model/filter.py

 
 FilterItem = namedtuple('FilterItem', 'type regexp')
 
-ACCEPTED_TYPES = set(Hit._fields)
+ACCEPTED_TYPES = set(Hit.FIELDS)
 
 class FilterList:
     def __init__(self):

core/model/visit.py

 
 VISIT_EXPIRATION_DELAY = timedelta(minutes=20)
 
-Hit = namedtuple('Hit', 'ip ident user time method url version response size referrer agent')
+class Hit:
+    FIELDS = ['ip', 'ident', 'user', 'time', 'method', 'url', 'version', 'response', 'size', 
+        'referrer', 'agent', 'sitename']
+    
+    def __init__(self, *args):
+        for fieldname, value in zip(self.FIELDS, args):
+            setattr(self, fieldname, value)
+    
+    @property
+    def url_with_sitename(self):
+        if self.sitename:
+            return '[{}]{}'.format(self.sitename, self.url)
+        else:
+            return self.url
+    
 
 class Visit:
     def __init__(self, basehit):

core/reader/apache.py

 import re
 from datetime import datetime
 
-from hsutil.files import FileOrPath
-
 from ..model.visit import Hit
 
 RE_LOGLINE = re.compile(r'(\d+\.\d+\.\d+\.\d+) ([^ ]*) ([^ ]*) \[([^ ]*) \+\d{4}\] "([^"]*)" (\d+) ([^ ]*) "([^"]*)" "([^"]*)"')
 DATETIME_FMT = '%d/%b/%Y:%H:%M:%S'
 
-def parse(infile):
-    with FileOrPath(infile, 'rt') as fp:
+def parse(logfile): # A ..conf.LogFile instance
+    with open(logfile.path, 'rt') as fp:
         for line in fp:
             m = RE_LOGLINE.match(line)
             if m is None:
                 method, url, version = request.split()
             except ValueError:
                 method, url, version = 'GET', '', 'HTTP/1.1'
-            yield Hit(ip, ident, user, time, method, url, version, response, size, referrer, agent)
+            yield Hit(ip, ident, user, time, method, url, version, response, size, referrer, agent,
+                logfile.sitename)
 
 class HitTableRow(Row):
     def __init__(self, hit):
-        self.url = hit.url
+        self.url = hit.url_with_sitename
         self.referrer = hit.referrer
         self.time = hit.time.strftime(DATETIME_FMT)
         self.agent = hit.agent

gui/main_window.py

         for logfile in self.conf.logfiles:
             self.load_log(logfile)
     
-    def load_log(self, logpath):
-        for hit in apache.parse(logpath):
+    def load_log(self, logfile):
+        for hit in apache.parse(logfile):
             if not self.conf.filters.hit_matches(hit):
                 self.visits.add_hit(hit)
         self.time_slider.refresh()

gui/visit_table.py

     def __init__(self, visit, hit):
         self.visit = visit
         self.ip = visit.ip
-        self.last_url = hit.url
+        self.last_url = hit.url_with_sitename
         self.referrer = visit.referrer
         self.last_time = hit.time.strftime(DATETIME_FMT)
         self.last_agent = hit.agent

qt/controller/main_window.py

     def _setupActions(self):
         # (name, shortcut, icon, desc, func)
         ACTIONS = [
-            ('actionLoadLog', 'Ctrl+L', '', "Load Log File...", self.loadLogTriggered),
-            ('actionLoadConfig', 'Ctrl+G', '', "Load Config File...", self.loadConfigTriggered),
+            ('actionLoadConfig', 'Ctrl+L', '', "Load Config File...", self.loadConfigTriggered),
         ]
         for name, shortcut, icon, desc, func in ACTIONS:
             action = QAction(self)
         self.menuHelp.setTitle("Help")
         self.setMenuBar(self.menubar)
         
-        self.menuFile.addAction(self.actionLoadLog)
         self.menuFile.addAction(self.actionLoadConfig)
         
         self.menubar.addAction(self.menuFile.menuAction())
         if confpath:
             self.model.load_conf(confpath)
     
-    def loadLogTriggered(self):
-        title = "Select a log file to load"
-        logpath = QFileDialog.getOpenFileName(self, title, '')
-        if logpath:
-            self.model.load_log(logpath)
-    
     return fp, logpath
 
 class TestConf(dict):
-    def add_testdata_log(self, logname):
+    def add_testdata_log(self, logname, sitename=None):
         logpath = TestData.filepath(logname)
-        self.add_log(logpath)
+        self.add_log(logpath, sitename)
     
-    def add_log_lines(self, lines):
+    def add_log_lines(self, lines, sitename=None):
         # this path is not always different on every call, so we have to find a temp filename in
         # that folder (which we don't have to clean because it's taken care of by pytest.
         fp, logpath = tmpfile()
         fp.write('\n'.join(lines))
         fp.close()
-        self.add_log(str(logpath))
+        self.add_log(str(logpath), sitename)
     
-    def add_log(self, path):
+    def add_log(self, path, sitename=None):
         if 'logfiles' not in self:
             self['logfiles'] = []
-        self['logfiles'].append(path)
+        logdict = {'path': path}
+        if sitename:
+            logdict['sitename'] = sitename
+        self['logfiles'].append(logdict)
     
     def add_filter(self, filter_type, value):
         if 'filters' not in self:

tests/main_test.py

     app.load_conf(conf)
     eq_(len(app.vtable), 1)
 
+def test_logfile_site_name():
+    # If a logfile has a specified site name, it will be a prefix to all urls shown from it
+    app, conf = TestApp(), TestConf()
+    lines = [
+        '1.1.1.1 - - [15/Nov/2010:04:20:00 +0000] "GET /foo HTTP/1.1" 200 0 "-" "-"'
+    ]
+    conf.add_log_lines(lines, sitename='site1')
+    lines = [
+        '1.1.1.1 - - [15/Nov/2010:04:25:00 +0000] "GET /bar HTTP/1.1" 200 0 "-" "-"' # same visit
+    ]
+    conf.add_log_lines(lines, sitename='site2')
+    app.load_conf(conf)
+    eq_(app.vtable[0].last_url, '[site2]/bar')
+    eq_(app.htable[0].url, '[site1]/foo')
+
 #---
 def app_load_simple_log():
     app, conf = TestApp(), TestConf()