Commits

Alexis Lê-Quôc  committed 391f120

First test passes.

  • Participants
  • Parent commits e9a3d9a

Comments (0)

Files changed (2)

File src/ApacheLogtailer.py

                     "%>s": r"(?P<final_retcode>\d{3})",
                     "%h": r"(?P<remote_host>\S+)",
                     "%l": r"(?P<remote_logname>\S+)",
-                    "%t": r"(?P<req_time_std>\[\w+ [+-]\d+\])",
+                    "%t": r"\[(?P<req_time_std>\S+ [+-]\d+)\]",
                     '"%r"': r'"(?P<request>[^"]+)"',
                     '"%{Referer}i"': r'"(?P<referer>[^"]+)"',
                     '"%{User-agent}i"': r'"(?P<user_agent>[^"]+)"',
         logger.debug("Log directive: %s\nRegex: %s" % (log_format, regex))
         return re.compile(regex)
     
-    def __init__(self, log_format='%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'):
+    def __init__(self, log_format='%h %l %u %t %D "%r" %>s %b "%{Referer}i" "%{User-Agent}i"'):
         '''This function should initialize any data structures or variables
         needed for the internal state of the line parser.
         log_format: apache log format string
         # assume we're in daemon mode unless set_check_duration gets called
         self.dur_override = False
 
+    def update_stats(self, regMatch):
+        linebits = regMatch.groupdict()
+        logger.debug("Matches: %s" % (linebits,))
+        assert "request" in linebits
+        # capture GETs
+        if( 'GET' in linebits.get("request", None) ):
+            self.num_gets+=1
+
+        # capture HTTP response code
+        rescode = linebits.get('init_retcode', None)
+
+        if rescode is None:
+            rescode = linebits.get('final_retcode', None)
+            
+        if rescode is not None:
+            rescode = float(rescode)
+            if( (rescode >= 200) and (rescode < 300) ):
+                self.num_two+=1
+            elif( (rescode >= 300) and (rescode < 400) ):
+                self.num_three+=1
+            elif( (rescode >= 400) and (rescode < 500) ):
+                self.num_four+=1
+            elif( (rescode >= 500) and (rescode < 600) ):
+                self.num_five+=1
+
+        # capture request duration
+        dur = float(linebits['req_time'])
+        # convert to seconds
+        dur = dur / 1000000
+        self.req_time += dur
+
+        # store for 90th % calculation
+        self.ninetieth.append(dur)
+        
+
     # example function for parse line
     # takes one argument (text) line to be parsed
     # returns nothing
         try:
             regMatch = self.reg.match(line)
             if regMatch:
-                linebits = regMatch.groupdict()
-                assert "request" in linebits
-                # capture GETs
-                if( 'GET' in linebits.get("request", None) ):
-                    self.num_gets+=1
-                # capture HTTP response code
-                rescode = float(linebits['init_retcode'])
-                if( (rescode >= 200) and (rescode < 300) ):
-                    self.num_two+=1
-                elif( (rescode >= 300) and (rescode < 400) ):
-                    self.num_three+=1
-                elif( (rescode >= 400) and (rescode < 500) ):
-                    self.num_four+=1
-                elif( (rescode >= 500) and (rescode < 600) ):
-                    self.num_five+=1
-                # capture request duration
-                dur = float(linebits['req_time'])
-                # convert to seconds
-                dur = dur / 1000000
-                self.req_time += dur
-                # store for 90th % calculation
-                self.ninetieth.append(dur)
+                self.update_stats(regMatch)
             else:
-                logger.warn("regmatch %s failed to match: %s" % (line, self.log_format))
-        except Exception, e:
+                # Sometimes \" sneaks in the line, fix it
+                if "\\\"" in line:
+                    line = line.replace("\\\"", "_")
+                    regMatch = self.reg.match(line)
+                    if regMatch:
+                        self.update_stats(regMatch)
+                    else:
+                        logger.warn("Regex\t%s failed to match:\n\t%s" % (self.reg.pattern, line))
+        finally:
             self.lock.release()
-            logger.exception(e)
-            raise LogtailerParsingException(e)
-        self.lock.release()
         
     # example function for deep copy
     # takes no arguments

File tests/test_apache.py

 import unittest
+import logging
+logging.basicConfig(format="%(asctime)s %(levelname)-8s %(message)s")
+
 from ApacheLogtailer import *
 
 class TestApacheLogtailer(unittest.TestCase):
         
     def testParseLine(self):
         for line in self.loglines:
-            hits = self.tailer.num_hits
             self.tailer.parse_line(line)
-            self.assertEquals(self.tailer.num_hits, hits + 1)
+        self.assertEquals(self.tailer.num_gets, 150103)
+        self.assertEquals(self.tailer.num_hits, 161004)
             
 if __name__ == "__main__":
     unittest.main()