Commits

Chris Perl committed 64b913f

Added some quick two level aggregators

Comments (0)

Files changed (1)

         self._key2_fmt = "\t%-100s %10d\n"
         super(HitsPerIPPerUrl, self)._aggregate(datetimeobj, url, ms, ip)
 
+class UrlAndIPByTimeToRenderLinearAggr(Aggregator):
+    def _aggregate(self, datetimeobj, url, ms, ip):
+        if self._re_filter and not self._re_filter.search(self._key1):
+            return
+        sub_aggr = self.aggr_data.setdefault(self._key1, {'hits': 0})
+        sub_aggr['hits'] += 1
+        sub_aggr2 = sub_aggr.setdefault(self._key2, {'hits': 0})
+        sub_aggr2['hits'] += 1
+        bucket_idx = ms / self._ms_step
+        bucket_start = self._ms_step * bucket_idx
+        bucket_end  = self._ms_step * (bucket_idx + 1)
+        hits_for_bucket = sub_aggr2.setdefault((bucket_start, bucket_end), 0)
+        sub_aggr2[(bucket_start, bucket_end)] = hits_for_bucket + 1
+
+    def display(self, display_obj):
+        sorted_keys = sorted(self.aggr_data, lambda a,b: cmp(self.aggr_data[a]['hits'], self.aggr_data[b]['hits']))
+        for key in sorted_keys:
+            display_obj.write(self._key1_fmt % (key, self.aggr_data[key]['hits']))
+            second_level_keys = [ x for x in self.aggr_data[key] if x != 'hits' ]
+            sorted_second_level_keys = sorted(second_level_keys, lambda a,b: cmp(self.aggr_data[key][a]['hits'], self.aggr_data[key][b]['hits']))
+            for second_key in sorted_second_level_keys:
+                display_obj.write(self._key2_fmt % (second_key, self.aggr_data[key][second_key]['hits']))
+                ms_period_keys = [ x for x in self.aggr_data[key][second_key] if x != 'hits' ]
+                sorted_ms_period_keys = sorted(ms_period_keys, lambda a,b: cmp(a[0], b[0]))
+                for bucket_bottom, bucket_top in sorted_ms_period_keys:
+                    display_obj.write("\t\t%4d ms - %4d ms %10d\n" % (bucket_bottom, bucket_top,
+                                                                    self.aggr_data[key][second_key][(bucket_bottom, bucket_top)]))
+                display_obj.write("\n")
+            display_obj.write("\n")
+
+
+class HitsPerUrlPerIPByTimeToRenderLinear(UrlAndIPByTimeToRenderLinearAggr):
+    # __call__ requires start, end, ms_step and re_filter keyword args
+    def _aggregate(self, datetimeobj, url, ms, ip):
+        # TODO: move this stuff into __init__
+        self._key1 = url
+        self._key2 = ip
+        self._key1_fmt = "%-100s %10d\n"
+        self._key2_fmt = "\t%16s %10d\n"
+        super(HitsPerUrlPerIPByTimeToRenderLinear, self)._aggregate(datetimeobj, url, ms, ip)
+
+class HitsPerIPPerUrlByTimeToRenderLinear(UrlAndIPByTimeToRenderLinearAggr):
+    # __call__ requires start, end, ms_step and re_filter keyword args
+    def _aggregate(self, datetimeobj, url, ms, ip):
+        # TODO: move this stuff into __init__
+        self._key1 = ip
+        self._key2 = url
+        self._key1_fmt = "%16s %10d\n"
+        self._key2_fmt = "\t%-100s %10d\n"
+        super(HitsPerIPPerUrlByTimeToRenderLinear, self)._aggregate(datetimeobj, url, ms, ip)
+
 class LogAnalyzer(object):
 
     month_map = {
         self._setup_parser_do_url_hits_per_time()
         self._setup_parser_do_url_time_to_render()
         self._setup_parser_do_url_hits_per_ip()
+        self._setup_parser_do_url_hits_per_ip_time_to_render()
         self._setup_parser_do_ip_hits_per_time()
         self._setup_parser_do_ip_time_to_render()
         self._setup_parser_do_ip_hits_per_url()
-
-        self._hpu      = HitsPerUrl(self)
-        self._hpi      = HitsPerIP(self)
-        self._hpuput   = HitsPerUrlPerUnitTime(self)
-        self._hpiput   = HitsPerIPPerUnitTime(self)
-        self._hpubttrl = HitsPerUrlByTimeToRenderLinear(self)
-        self._hpibttrl = HitsPerIPByTimeToRenderLinear(self)
-        self._hpupi    = HitsPerUrlPerIP(self)
-        self._hpipu    = HitsPerIPPerUrl(self)
+        self._setup_parser_do_ip_hits_per_url_time_to_render()
+
+        self._hpu        = HitsPerUrl(self)
+        self._hpi        = HitsPerIP(self)
+        self._hpuput     = HitsPerUrlPerUnitTime(self)
+        self._hpiput     = HitsPerIPPerUnitTime(self)
+        self._hpubttrl   = HitsPerUrlByTimeToRenderLinear(self)
+        self._hpibttrl   = HitsPerIPByTimeToRenderLinear(self)
+        self._hpupi      = HitsPerUrlPerIP(self)
+        self._hpipu      = HitsPerIPPerUrl(self)
+        self._hpupibttrl = HitsPerUrlPerIPByTimeToRenderLinear(self)
+        self._hpipubttrl = HitsPerIPPerUrlByTimeToRenderLinear(self)
 
     def cmd_loop(self):
         while True:
         self._parser_do_url_hits_per_ip = NonExitingOptionParser(usage="""url-hits-per-ip --filter FILTER""")
         self._parser_do_url_hits_per_ip.add_option('-f', '--filter', action='store', type='str', dest='re_filter')
 
+    def _setup_parser_do_url_hits_per_ip_time_to_render(self):
+        self._parser_do_url_hits_per_ip_time_to_render = NonExitingOptionParser(usage="""url-hits-per-ip-time-to-render --step STEP --filter FILTER""")
+        self._parser_do_url_hits_per_ip_time_to_render.add_option('-s', '--step', action='store', type='int', dest='step')
+        self._parser_do_url_hits_per_ip_time_to_render.add_option('-f', '--filter', action='store', type='str', dest='re_filter')
+
     def _setup_parser_do_toptalkers(self):
         self._parser_do_toptalkers = NonExitingOptionParser(usage="toptalkers [number of IPs to display]")
         self._parser_do_toptalkers.add_option('-f', '--filter', action='store', type='str', dest='re_filter')
         self._parser_do_ip_hits_per_url = NonExitingOptionParser(usage="""ip-hits-per-url --filter FILTER""")
         self._parser_do_ip_hits_per_url.add_option('-f', '--filter', action='store', type='str', dest='re_filter')
 
+    def _setup_parser_do_ip_hits_per_url_time_to_render(self):
+        self._parser_do_ip_hits_per_url_time_to_render = NonExitingOptionParser(usage="""ip-hits-per-url-time-to-render --step STEP --filter FILTER""")
+        self._parser_do_ip_hits_per_url_time_to_render.add_option('-s', '--step', action='store', type='int', dest='step')
+        self._parser_do_ip_hits_per_url_time_to_render.add_option('-f', '--filter', action='store', type='str', dest='re_filter')
+
     def _do_set(self, *cmd_args):
         options, args = self._parser_do_set.parse_args(list(cmd_args))
         if len(args) != 2:
         self._hpipu(start=self.start, end=self.end, re_filter=options.re_filter)
         self._hpipu.display(self._display_obj)
 
+    def _do_url_hits_per_ip_time_to_render(self, *cmd_args):
+        options, args = self._parser_do_url_hits_per_ip_time_to_render.parse_args(list(cmd_args))
+        if not options.step:
+            step = 500
+        else:
+            step = options.step
+        self._hpupibttrl(start=self.start, end=self.end, ms_step=step, re_filter=options.re_filter)
+        self._hpupibttrl.display(self._display_obj)
+
+    def _do_ip_hits_per_url_time_to_render(self, *cmd_args):
+        options, args = self._parser_do_ip_hits_per_url_time_to_render.parse_args(list(cmd_args))
+        if not options.step:
+            step = 500
+        else:
+            step = options.step
+        self._hpipubttrl(start=self.start, end=self.end, ms_step=step, re_filter=options.re_filter)
+        self._hpipubttrl.display(self._display_obj)
+
 def main():
     log_analyzer = LogAnalyzer()
     for path in sys.argv[1:]: