György Kohut avatar György Kohut committed c60d8f7

add inital binaries stats page

Comments (0)

Files changed (7)

hbwebui/queries.py

+from django.db import connections
+
+class QueryBase(object):
+    def __init__(self):
+        self.map = {}
+        self.list = []
+        self.cursor = None
+    
+    def part_init(self, part):
+        self.map[part] = []
+        self.map[part].append(set()) # [0]: set of subpart names, to filter duplicates, trigger insertion
+        self.map[part].append('') # [1]: query fragment text
+        self.map[part].append([]) # [2]: query params
+        self.list.append(part)
+    
+    def part_append(self, part, subpart, sep, text, params):
+        if subpart in self.map[part][0]: # filter duplicate subparts
+            return False
+        else:
+            if len(self.map[part][0]):
+                self.map[part][1] += sep + text
+            else:
+                self.map[part][1] += text
+            self.map[part][2].extend(params)
+            self.map[part][0].add(subpart)
+            return True
+    
+    def part_append2(self, part, text):
+        self.map[part][1] += text
+    
+    def part_init2(self, part, text):
+        self.part_init(part)
+        self.part_append2(part, text)
+        
+    def where_init(self):
+        self.part_init2('where_clause', 'WHERE ')
+    
+    def order_by_init(self):
+        self.part_init2('order_by_clause', 'ORDER BY ')
+    
+    def with_init(self):
+        self.part_init2('with_clause', 'WITH ')
+        self.part_init2('with_tables', ', ')
+    
+    def limit_init(self):
+        self.part_init2('limit_clause', 'LIMIT ')
+    
+    def offset_init(self):
+        self.part_init2('offset_clause', 'OFFSET ')
+    
+    def where_append(self, subpart, text, params):
+        return self.part_append('where_clause', subpart, ' AND ', text, params)
+    
+    def order_by_append(self, subpart, text, desc=False):
+        return self.part_append('order_by_clause', subpart, ', ', text + (' DESC' if desc else ''), [])
+    
+    def with_append(self, subpart, query, table, params):
+        b = self.part_append('with_clause', subpart, ', ', query, params)
+        if b:
+            self.part_append('with_tables', subpart, ', ', table, [])
+            return True
+        else:
+            return False
+    
+    def limit(self, limit):
+        return self.part_append('limit_clause', 'limit_clause', '', '%s', [limit])
+    
+    def offset(self, offset):
+        return self.part_append('offset_clause', 'offset_clause', '', '%s', [offset])
+    
+    def build(self):
+        query = ''
+        params = []
+        format_params = {}
+        
+        for part in self.list:
+            if len(self.map[part][0]): # got subparts
+                format_params[part] = self.map[part][1]
+                params.extend(self.map[part][2])
+            else:
+                format_params[part] = ''
+        
+        query = self.tpl % format_params
+        return [query, params]
+    
+    def execute(self):
+        self.cursor = connections[self.db].cursor()
+        query, params = self.build()
+        
+        self.cursor.execute(query, params)
+        return self.cursor
+
+
+class Stats_Binaries(QueryBase):
+    db = 'hbstats'
+    tpl = \
+        """SELECT
+                md5, sum(agg_main.n_count)::bigint as n_count,
+                count(DISTINCT ident_id) AS n_ident, count(DISTINCT source_ip) AS n_source_ip, count(DISTINCT target_port) AS n_target_port,
+                max(agg_main.ts_last) as ts_last, min(agg_main.ts_first) as ts_first
+            FROM
+                agg_main
+            JOIN
+                dim_ident ON agg_main.ident_id = dim_ident.id
+            JOIN
+                dim_binary ON agg_main.binary_id = dim_binary.id
+            %(where_clause)s
+            GROUP BY md5
+            %(order_by_clause)s"""
+    
+    def __init__(self):
+        super(Stats_Binaries, self).__init__()
+        self.where_init()
+        self.order_by_init()
+    
+    def filter_ident(self, idents):
+        self.where_append('filter_ident', 'ident_id IN (SELECT id from dim_ident WHERE ident IN %s)', [tuple(idents)])
+    
+    def filter_source_ip(self, source_ips):
+        self.where_append('filter_source_ip', 'source_ip IN %s', [tuple(source_ips)])
+    
+    def order_n_count(self, desc=False):
+        self.order_by_append('order_n_count', 'n_count', desc)
+
+
+class Stats_Binaries_TimeRange(QueryBase):
+    db = 'hbstats'
+    tpl = \
+        """%(with_clause)s
+        
+        SELECT
+            md5, sum(t.n_count)::bigint as n_count,
+            count(DISTINCT ident_id) AS n_ident, count(DISTINCT source_ip) AS n_source_ip, count(DISTINCT target_port) AS n_target_port,
+            max(t.ts_last) AS ts_last, min(t.ts_first) AS ts_first
+        FROM
+            (SELECT
+                agg_id, sum(n_count) as n_count, max(ts_min) as ts_last, min(ts_min) as ts_first
+                FROM ts_main_min
+                WHERE agg_id IN (SELECT agg_id FROM agg_main %(where_clause)s)
+                GROUP BY agg_id
+            ) as t  %(with_tables)s
+        JOIN
+            agg_main ON t.agg_id = agg_main.agg_id
+        JOIN
+            dim_ident ON agg_main.ident_id = dim_ident.id
+        JOIN
+            dim_binary ON agg_main.binary_id = dim_binary.id
+        GROUP BY md5
+        %(order_by_clause)s"""
+    
+    def __init__(self):
+        super(Stats_Binaries_TimeRange, self).__init__()
+        self.with_init()
+        self.where_init()
+        self.order_by_init()
+        
+    def filter_time_range_recent(self, hours):
+        b = self.where_append('filter_time_range', 'ts_min <= tr_end AND ts_min > tr_begin', [begin, end])
+        if b:
+            self.with_append('filter_time_range',
+                "tr AS SELECT max(ts_min) AS tr_end, max(ts_min) - interval '%s hour' AS tr_begin from ts_main_min)",
+                'tr',
+                [hours]
+            )
+    
+    def filter_time_range(self, begin, end):
+        self.where_append('filter_time_range', 'ts_min >= %s AND ts_min <= %s', [begin, end])
+    
+    def filter_ident(self, idents):
+        self.where_append('filter_ident', 'ident_id IN (SELECT id from dim_ident WHERE ident IN %s)', [tuple(idents)])
+    
+    def filter_source_ip(self, source_ips):
+        self.where_append('filter_source_ip', 'source_ip IN %s', [tuple(source_ips)])
+    
+    def order_n_count(self, desc=False):
+        self.order_by_append('order_n_count', 'n_count', desc)
+
+
+class Stats_Binaries_Summary(QueryBase):
+    db = 'hbstats'
+    tpl = \
+        """SELECT
+                count(DISTINCT binary_id) as n_count,
+                count(DISTINCT ident_id) AS n_ident, count(DISTINCT source_ip) AS n_source_ip, count(DISTINCT target_port) AS n_target_port,
+                max(agg_main.ts_last) as ts_last, min(agg_main.ts_first) as ts_first,
+                sum(n_count)::bigint as n_total
+            FROM
+                agg_main
+            %(where_clause)s"""
+    
+    def __init__(self):
+        super(Stats_Binaries_Summary, self).__init__()
+        self.where_init()
+    
+    def filter_ident(self, idents):
+        self.where_append('filter_ident', 'ident_id IN (SELECT id from dim_ident WHERE ident IN %s)', [tuple(idents)])
+    
+    def filter_source_ip(self, source_ips):
+        self.where_append('filter_source_ip', 'source_ip IN %s', [tuple(source_ips)])

hbwebui/static/js/lodash.min.js

+/*!
+ Lo-Dash 0.4.2 lodash.com/license
+ Underscore.js 1.3.3 github.com/documentcloud/underscore/blob/master/LICENSE
+*/
+;(function(e,t){"use strict";function s(e){return new o(e)}function o(e){if(e&&e._wrapped)return e;this._wrapped=e}function u(e,t,n){t||(t=0);var r=e.length,i=r-t>=(n||30),s=i?{}:e;if(i)for(var o=t-1;++o<r;)n=e[o]+"",(Y.call(s,n)?s[n]:s[n]=[]).push(e[o]);return function(e){if(i){var n=e+"";return Y.call(s,n)&&-1<E(s[n],e)}return-1<E(s,e,t)}}function a(){for(var e,t,n,s=-1,o=arguments.length,u={e:"",f:"",j:"",p:"",c:{d:""},m:{d:""}};++s<o;)for(t in e=arguments[s],e)n=(n=e[t])==r?"":n,/d|i/.test(t)?
+("string"==typeof n&&(n={b:n,l:n}),u.c[t]=n.b,u.m[t]=n.l):u[t]=n;e=u.a,t=/^[^,]+/.exec(e)[0],u.g=t,u.h=mt,u.k=wt,u.o=J,u.q=u.q!==i,u.r=u.r!==i,"n"in u||(u.n=yt),u.f||(u.f="if(!"+t+")return D");if("g"!=t||!u.c.i)u.c=r;t="",u.r&&(t+="'use strict';"),t+="var n,p="+u.g+",D",u.j&&(t+="="+u.j),t+=";"+u.f+";"+u.p+";",u.c&&(t+="var s=p.length;n=-1;",u.m&&(t+="if(s===s>>>0){"),u.n&&(t+="if(J.call(p)==G){p=p.split('')}"),t+=u.c.d+";while(++n<s){"+u.c.i+"}",u.m&&(t+="}"));if(u.m){u.c&&(t+="else{"),u.h||(t+="var E=typeof p=='function'&&C.call(p,'prototype');"
+),u.k&&u.q?t+="var A=u(p),z=-1,s=A.length;"+u.m.d+";while(++z<s){n=A[z];if(!(E&&n=='prototype')){"+u.m.i+"}}":(t+=u.m.d+";for(n in p){",u.h?(u.q&&(t+="if(l.call(p,n)){"),t+=u.m.i+";",u.q&&(t+="}")):(t+="if(!(E&&n=='prototype')",u.q&&(t+="&&l.call(p,n)"),t+="){"+u.m.i+"}"),t+="}");if(u.h){t+="var i=p.constructor;";for(n=0;7>n;n++)t+="n='"+u.o[n]+"';if(","constructor"==u.o[n]&&(t+="!(i&&i.prototype===p)&&"),t+="l.call(p,n)){"+u.m.i+"}"}u.c&&(t+="}")}return t+=u.e+";return D",Function("c,d,h,j,l,m,r,x,u,C,F,G,J"
+,"return function("+e+"){"+t+"}")(ut,C,f,lt,Y,L,p,St,ot,et,tt,pt,nt)}function f(e,n){return e=e.a,n=n.a,e===t?1:n===t?-1:e<n?-1:e>n?1:0}function l(e,t){return Q[t]}function c(e){return"\\"+xt[e]}function h(e){return Et[e]}function p(e,t){return function(n,r,i){return e.call(t,n,r,i)}}function d(){}function v(e,t){if(I.test(t))return"<e%-"+t+"%>";var n=Q.length;return Q[n]="'+__e("+t+")+'",K+n}function m(e,t,n,r){return e=Q.length,t?Q[e]="';"+t+";__p+='":n?Q[e]="'+__e("+n+")+'":r&&(Q[e]="'+((__t=("+
+r+"))==null?'':__t)+'"),K+e}function g(e,t){if(I.test(t))return"<e%="+t+"%>";var n=Q.length;return Q[n]="'+((__t=("+t+"))==null?'':__t)+'",K+n}function y(e,t,n,r){if(!e)return n;var i=e.length,s=3>arguments.length;r&&(t=p(t,r));if(i===i>>>0){var o=yt&&nt.call(e)==pt?e.split(""):e;for(i&&s&&(n=o[--i]);i--;)n=t(n,o[i],i,e);return n}o=Zt(e);for((i=o.length)&&s&&(n=e[o[--i]]);i--;)s=o[i],n=t(n,e[s],s,e);return n}function b(e,t,n){if(e)return t==r||n?e[0]:tt.call(e,0,t)}function w(e,t){var n=[];if(!e)
+return n;for(var r,i=-1,s=e.length;++i<s;)r=e[i],Gt(r)?Z.apply(n,t?r:w(r)):n.push(r);return n}function E(e,t,n){if(!e)return-1;var r=-1,i=e.length;if(n){if("number"!=typeof n)return r=T(e,t),e[r]===t?r:-1;r=(0>n?Math.max(0,i+n):n)-1}for(;++r<i;)if(e[r]===t)return r;return-1}function S(e,t,n){var r=-Infinity,i=r;if(!e)return i;var s=-1,o=e.length;if(!t){for(;++s<o;)e[s]>i&&(i=e[s]);return i}for(n&&(t=p(t,n));++s<o;)n=t(e[s],s,e),n>r&&(r=n,i=e[s]);return i}function x(e,t,n){return e?tt.call(e,t==r||
+n?1:t):[]}function T(e,t,n,r){if(!e)return 0;var i=0,s=e.length;if(n){r&&(n=C(n,r));for(t=n(t);i<s;)r=i+s>>>1,n(e[r])<t?i=r+1:s=r}else for(;i<s;)r=i+s>>>1,e[r]<t?i=r+1:s=r;return i}function N(e,t,n,r){var s=[];if(!e)return s;var o=-1,u=e.length,a=[];"function"==typeof t&&(r=n,n=t,t=i);for(n?r&&(n=p(n,r)):n=L;++o<u;)if(r=n(e[o],o,e),t?!o||a[a.length-1]!==r:0>E(a,r))a.push(r),s.push(e[o]);return s}function C(e,t){function n(){var o=arguments,u=t;return i||(e=t[r]),s.length&&(o=o.length?G.apply(s,o)
+:s),this instanceof n?(d.prototype=e.prototype,u=new d,(o=e.apply(u,o))&&St[typeof o]?o:u):e.apply(u,o)}var r,i=nt.call(e)==lt;if(i){if(bt||rt&&2<arguments.length)return rt.call.apply(rt,arguments)}else r=t,t=e;var s=tt.call(arguments,2);return n}function k(e,t,s){s||(s=[]);if(e===t)return 0!==e||1/e==1/t;if(e==r||t==r)return e===t;e._chain&&(e=e._wrapped),t._chain&&(t=t._wrapped);if(e.isEqual&&nt.call(e.isEqual)==lt)return e.isEqual(t);if(t.isEqual&&nt.call(t.isEqual)==lt)return t.isEqual(e);var o=
+nt.call(e);if(o!=nt.call(t))return i;switch(o){case pt:return e==""+t;case ct:return e!=+e?t!=+t:0==e?1/e==1/t:e==+t;case at:case ft:return+e==+t;case ht:return e.source==t.source&&e.global==t.global&&e.multiline==t.multiline&&e.ignoreCase==t.ignoreCase}if("object"!=typeof e||"object"!=typeof t)return i;for(var u=s.length;u--;)if(s[u]==e)return n;var u=-1,a=n,f=0;s.push(e);if(o==ut){if(f=e.length,a=f==t.length)for(;f--&&(a=k(e[f],t[f],s)););}else{if("constructor"in e!="constructor"in t||e.constructor!=
+t.constructor)return i;for(var l in e)if(Y.call(e,l)&&(f++,!(a=Y.call(t,l)&&k(e[l],t[l],s))))break;if(a){for(l in t)if(Y.call(t,l)&&!(f--))break;a=!f}if(a&&mt)for(;7>++u&&(l=J[u],!Y.call(e,l)||!!(a=Y.call(t,l)&&k(e[l],t[l],s))););}return s.pop(),a}function L(e){return e}function A(e){Bt(Qt(e),function(t){var r=s[t]=e[t];o.prototype[t]=function(){var e=[this._wrapped];return arguments.length&&Z.apply(e,arguments),e=r.apply(s,e),this._chain&&(e=new o(e),e._chain=n),e}})}var n=!0,r=null,i=!1,O,M,_,D
+,P="object"==typeof exports&&exports&&("object"==typeof global&&global&&global==global.global&&(e=global),exports),H=Array.prototype,B=Object.prototype,j=0,F=e._,I=/[-+=!~*%&^<>|{(\/]|\[\D|\b(?:delete|in|instanceof|new|typeof|void)\b/,q=/\b__p\+='';/g,R=/\b(__p\+=)''\+/g,U=/(__e\(.*?\)|\b__t\))\+'';/g,z=/(?:__e|__t=)\(\s*(?![\d\s"']|this\.)/g,W=RegExp("^"+(B.valueOf+"").replace(/[.*+?^=!:${}()|[\]\/\\]/g,"\\$&").replace(/valueOf|for [^\]]+/g,".+?")+"$"),X=/__token__(\d+)/g,V=/[&<"']/g,$=/['\n\r\t\u2028\u2029\\]/g
+,J="constructor hasOwnProperty isPrototypeOf propertyIsEnumerable toLocaleString toString valueOf".split(" "),K="__token__",Q=[],G=H.concat,Y=B.hasOwnProperty,Z=H.push,et=B.propertyIsEnumerable,tt=H.slice,nt=B.toString,rt=W.test(rt=tt.bind)&&rt,it=W.test(it=Array.isArray)&&it,st=e.isFinite,ot=W.test(ot=Object.keys)&&ot,ut="[object Array]",at="[object Boolean]",ft="[object Date]",lt="[object Function]",ct="[object Number]",ht="[object RegExp]",pt="[object String]",dt=e.clearTimeout,vt=e.setTimeout
+,mt=!et.call({valueOf:0},"valueOf"),gt="x"!=tt.call("x")[0],yt="xx"!="x"[0]+Object("x")[0],bt=rt&&/\n|Opera/.test(rt+nt.call(e.opera)),wt=ot&&/^.+$|true/.test(ot+!!e.attachEvent),Et={"&":"&amp;","<":"&lt;",'"':"&quot;","'":"&#x27;"},St={"boolean":i,"function":n,object:n,number:i,string:i,"undefined":i},xt={"\\":"\\","'":"'","\n":"n","\r":"r","	":"t","\u2028":"u2028","\u2029":"u2029"};s.templateSettings={escape:/<%-([\s\S]+?)%>/g,evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,variable:"obj"
+};var Tt={a:"g,e,I",j:"g",p:"if(!e){e=m}else if(I)e=r(e,I)",i:"e(p[n],n,g)"},Nt={j:"true",i:"if(!e(p[n],n,g))return!D"},Ct={q:i,r:i,a:"w",j:"w",p:"for(var q=1,s=arguments.length;q<s;q++){p=arguments[q];"+(mt?"if(p){":""),i:"D[n]=p[n]",e:(mt?"}":"")+"}"},kt={j:"[]",i:"e(p[n],n,g)&&D.push(p[n])"},Lt={p:"if(I)e=r(e,I)"},At={i:{l:Tt.i}},Ot={j:"",f:"if(!g)return[]",d:{b:"D=Array(s)",l:"D="+(wt?"Array(s)":"[]")},i:{b:"D[n]=e(p[n],n,g)",l:"D"+(wt?"[z]=":".push")+"(e(p[n],n,g))"}},Mt=a({a:"w",f:"if(!(w&&x[typeof w]))throw TypeError()"
+,j:"[]",i:"D.push(n)"}),_t=a({a:"g,H",j:"false",n:i,d:{b:"if(J.call(p)==G)return g.indexOf(H)>-1"},i:"if(p[n]===H)return true"}),Dt=a(Tt,Nt),Pt=a(Tt,kt),Ht=a(Tt,Lt,{j:"",i:"if(e(p[n],n,g))return p[n]"}),Bt=a(Tt,Lt),jt=a(Tt,{j:"{}",p:"var y,o=typeof e=='function';if(o&&I)e=r(e,I)",i:"y=o?e(p[n],n,g):p[n][e];(l.call(D,y)?D[y]:D[y]=[]).push(p[n])"}),Ft=a(Ot,{a:"g,t",p:"var b=F.call(arguments,2),o=typeof t=='function'",i:{b:"D[n]=(o?t:p[n][t]).apply(p[n],b)",l:"D"+(wt?"[z]=":".push")+"((o?t:p[n][t]).apply(p[n],b))"
+}}),It=a(Tt,Ot),qt=a(Ot,{a:"g,B",i:{b:"D[n]=p[n][B]",l:"D"+(wt?"[z]=":".push")+"(p[n][B])"}}),Rt=a({a:"g,e,a,I",j:"a",p:"var v=arguments.length<3;if(I)e=r(e,I)",d:{b:"if(v)D=g[++n]"},i:{b:"D=e(D,p[n],n,g)",l:"D=v?(v=false,p[n]):e(D,p[n],n,g)"}}),Ut=a(Tt,kt,{i:"!"+kt.i}),zt=a(Tt,Nt,{j:"false",i:Nt.i.replace("!","")}),Wt=a(Tt,Ot,{p:"if(typeof e=='string'){var y=e;e=function(g){return g[y]}}else if(I)e=r(e,I)",i:{b:"D[n]={a:e(p[n],n,g),b:p[n]}",l:"D"+(wt?"[z]=":".push")+"({a:e(p[n],n,g),b:p[n]})"},e
+:"D.sort(h);s=D.length;while(s--){D[s]=D[s].b}"}),Xt=a({q:i,r:i,a:"w",j:"w",p:"var k=arguments,s=k.length;if(s>1){for(var n=1;n<s;n++)D[k[n]]=d(D[k[n]],D);return D}",i:"if(J.call(D[n])==j)D[n]=d(D[n],D)"}),Vt=a(Ct,{i:"if(D[n]==null)"+Ct.i}),$t=a(Ct),Jt=a(Tt,Lt,At,{q:i}),Kt=a(Tt,Lt,At),Qt=a({q:i,a:"w",j:"[]",i:"if(J.call(p[n])==j)D.push(n)",e:"D.sort()"});Bt({Arguments:"[object Arguments]",Date:ft,Function:lt,Number:ct,RegExp:ht,String:pt},function(e,t){s["is"+t]=function(t){return nt.call(t)==e}}
+),s.isArguments(arguments)||(s.isArguments=function(e){return!!e&&!!Y.call(e,"callee")});var Gt=it||function(e){return nt.call(e)==ut},Yt=a({a:"K",j:"true",p:"var f=J.call(K);if(f==c||f==G)return!K.length",i:{l:"return false"}}),Zt=ot?function(e){return"function"==typeof e&&et.call(e,"prototype")?Mt(e):ot(e)}:Mt,en=a({a:"w",j:"[]",i:"D.push(p[n])"});s.VERSION="0.4.2",s.after=function(e,t){return 1>e?t():function(){if(1>--e)return t.apply(this,arguments)}},s.bind=C,s.bindAll=Xt,s.chain=function(e)
+{return e=new o(e),e._chain=n,e},s.clone=function(e){return e&&St[typeof e]?Gt(e)?e.slice():$t({},e):e},s.compact=function(e){var t=[];if(!e)return t;for(var n=-1,r=e.length;++n<r;)e[n]&&t.push(e[n]);return t},s.compose=function(){var e=arguments;return function(){for(var t=arguments,n=e.length;n--;)t=[e[n].apply(this,t)];return t[0]}},s.contains=_t,s.debounce=function(e,t,n){function i(){a=r,n||e.apply(u,s)}var s,o,u,a;return function(){var r=n&&!a;return s=arguments,u=this,dt(a),a=vt(i,t),r&&(o=
+e.apply(u,s)),o}},s.defaults=Vt,s.defer=function(e){var n=tt.call(arguments,1);return vt(function(){return e.apply(t,n)},1)},s.delay=function(e,n){var r=tt.call(arguments,2);return vt(function(){return e.apply(t,r)},n)},s.difference=function(e){var t=[];if(!e)return t;for(var n=-1,r=e.length,i=G.apply(t,arguments),i=u(i,r);++n<r;)i(e[n])||t.push(e[n]);return t},s.escape=function(e){return e==r?"":(e+"").replace(V,h)},s.every=Dt,s.extend=$t,s.filter=Pt,s.find=Ht,s.first=b,s.flatten=w,s.forEach=Bt,
+s.forIn=Jt,s.forOwn=Kt,s.functions=Qt,s.groupBy=jt,s.has=function(e,t){return Y.call(e,t)},s.identity=L,s.indexOf=E,s.initial=function(e,t,n){return e?tt.call(e,0,-(t==r||n?1:t)):[]},s.intersection=function(e){var t=[];if(!e)return t;for(var n,r=-1,i=e.length,s=tt.call(arguments,1),o=[];++r<i;)n=e[r],0>E(t,n)&&Dt(s,function(e,t){return(o[t]||(o[t]=u(e)))(n)})&&t.push(n);return t},s.invoke=Ft,s.isArray=Gt,s.isBoolean=function(e){return e===n||e===i||nt.call(e)==at},s.isElement=function(e){return!!
+e&&1==e.nodeType},s.isEmpty=Yt,s.isEqual=k,s.isFinite=function(e){return st(e)&&nt.call(e)==ct},s.isNaN=function(e){return nt.call(e)==ct&&e!=+e},s.isNull=function(e){return e===r},s.isObject=function(e){return e&&St[typeof e]},s.isUndefined=function(e){return e===t},s.keys=Zt,s.last=function(e,t,n){if(e){var i=e.length;return t==r||n?e[i-1]:tt.call(e,-t||i)}},s.lastIndexOf=function(e,t,n){if(!e)return-1;var r=e.length;for(n&&"number"==typeof n&&(r=(0>n?Math.max(0,r+n):Math.min(n,r-1))+1);r--;)if(
+e[r]===t)return r;return-1},s.map=It,s.max=S,s.memoize=function(e,t){var n={};return function(){var r=t?t.apply(this,arguments):arguments[0];return Y.call(n,r)?n[r]:n[r]=e.apply(this,arguments)}},s.min=function(e,t,n){var r=Infinity,i=r;if(!e)return i;var s=-1,o=e.length;if(!t){for(;++s<o;)e[s]<i&&(i=e[s]);return i}for(n&&(t=p(t,n));++s<o;)n=t(e[s],s,e),n<r&&(r=n,i=e[s]);return i},s.mixin=A,s.noConflict=function(){return e._=F,this},s.once=function(e){var t,r=i;return function(){return r?t:(r=n,t=
+e.apply(this,arguments))}},s.partial=function(e){var t=tt.call(arguments,1),n=t.length;return function(){var r;return r=arguments,r.length&&(t.length=n,Z.apply(t,r)),r=1==t.length?e.call(this,t[0]):e.apply(this,t),t.length=n,r}},s.pick=function(e){for(var t,n=0,r=G.apply(H,arguments),i=r.length,s={};++n<i;)t=r[n],t in e&&(s[t]=e[t]);return s},s.pluck=qt,s.range=function(e,t,n){n||(n=1),t==r&&(t=e||0,e=0);for(var i=-1,t=Math.max(0,Math.ceil((t-e)/n)),s=Array(t);++i<t;)s[i]=e,e+=n;return s},s.reduce=
+Rt,s.reduceRight=y,s.reject=Ut,s.rest=x,s.result=function(e,t){if(!e)return r;var n=e[t];return nt.call(n)==lt?e[t]():n},s.shuffle=function(e){if(!e)return[];for(var t,n=-1,r=e.length,i=Array(r);++n<r;)t=Math.floor(Math.random()*(n+1)),i[n]=i[t],i[t]=e[n];return i},s.size=function(e){if(!e)return 0;var t=e.length;return t===t>>>0?e.length:Zt(e).length},s.some=zt,s.sortBy=Wt,s.sortedIndex=T,s.tap=function(e,t){return t(e),e},s.template=function(e,t,n){n||(n={});var i,o;i=n.escape;var u=n.evaluate,
+a=n.interpolate,f=s.templateSettings,n=n.variable;i==r&&(i=f.escape),u==r&&(u=f.evaluate),a==r&&(a=f.interpolate),i&&(e=e.replace(i,v)),a&&(e=e.replace(a,g)),u!=O&&(O=u,D=RegExp((u?u.source:"($^)")+"|<e%-([\\s\\S]+?)%>|<e%=([\\s\\S]+?)%>","g")),i=Q.length,e=e.replace(D,m),i=i!=Q.length,e="__p += '"+e.replace($,c).replace(X,l)+"';",Q.length=0,n||(n=f.variable||M||"obj",i?e="with("+n+"){"+e+"}":(n!=M&&(M=n,_=RegExp("(\\(\\s*)"+n+"\\."+n+"\\b","g")),e=e.replace(z,"$&"+n+".").replace(_,"$1__d"))),e=(
+i?e.replace(q,""):e).replace(R,"$1").replace(U,"$1;"),e="function("+n+"){"+n+"||("+n+"={});var __t,__p='',__e=_.escape"+(i?",__j=Array.prototype.join;function print(){__p+=__j.call(arguments,'')}":",__d="+n+"."+n+"||"+n+";")+e+"return __p}";try{o=Function("_","return "+e)(s)}catch(h){o=function(){throw h}}return t?o(t):(o.source=e,o)},s.throttle=function(e,t){function n(){a=new Date,u=r,e.apply(o,i)}var i,s,o,u,a=0;return function(){var r=new Date,f=t-(r-a);return i=arguments,o=this,0>=f?(a=r,s=e
+.apply(o,i)):u||(u=vt(n,f)),s}},s.times=function(e,t,n){var r=-1;if(n)for(;++r<e;)t.call(n,r);else for(;++r<e;)t(r)},s.toArray=function(e){if(!e)return[];if(e.toArray&&nt.call(e.toArray)==lt)return e.toArray();var t=e.length;return t===t>>>0?(gt?nt.call(e)==pt:"string"==typeof e)?e.split(""):tt.call(e):en(e)},s.union=function(){for(var e=-1,t=[],n=G.apply(t,arguments),r=n.length;++e<r;)0>E(t,n[e])&&t.push(n[e]);return t},s.uniq=N,s.uniqueId=function(e){var t=j++;return e?e+t:t},s.values=en,s.without=
+function(e){var t=[];if(!e)return t;for(var n=-1,r=e.length,i=u(arguments,1,20);++n<r;)i(e[n])||t.push(e[n]);return t},s.wrap=function(e,t){return function(){var n=[e];return arguments.length&&Z.apply(n,arguments),t.apply(this,n)}},s.zip=function(e){if(!e)return[];for(var t=-1,n=S(qt(arguments,"length")),r=Array(n);++t<n;)r[t]=qt(arguments,t);return r},s.zipObject=function(e,t){if(!e)return{};var n=-1,r=e.length,i={};for(t||(t=[]);++n<r;)i[e[n]]=t[n];return i},s.all=Dt,s.any=zt,s.collect=It,s.detect=
+Ht,s.each=Bt,s.foldl=Rt,s.foldr=y,s.head=b,s.include=_t,s.inject=Rt,s.methods=Qt,s.select=Pt,s.tail=x,s.take=b,s.unique=N,o.prototype=s.prototype,A(s),o.prototype.chain=function(){return this._chain=n,this},o.prototype.value=function(){return this._wrapped},Bt("pop push reverse shift sort splice unshift".split(" "),function(e){var t=H[e];o.prototype[e]=function(){var e=this._wrapped;return t.apply(e,arguments),e.length===0&&delete e[0],this._chain&&(e=new o(e),e._chain=n),e}}),Bt(["concat","join"
+,"slice"],function(e){var t=H[e];o.prototype[e]=function(){var e=t.apply(this._wrapped,arguments);return this._chain&&(e=new o(e),e._chain=n),e}}),typeof define=="function"&&typeof define.amd=="object"&&define.amd?(e._=s,define(function(){return s})):P?"object"==typeof module&&module&&module.s==P?(module.s=s)._=s:P._=s:e._=s})(this);

hbwebui/templates/navbar.html

         <div class="container-fluid">
             <ul class="nav">
                 <li {% if page == "dashboard" %}class="active"{% endif %}><a href="{% if page == "dashboard" %}#{% else %}{% url dashboard %}{% endif %}">Dashboard</a></li>
-                <li class="dropdown">
-                    <a class="dropdown-toggle" data-toggle="dropdown" href="#">Stats <span class="caret"></span></a>
+                
+                {% if page == "stats_binaries" %}
+                <li class="dropdown active">
+                    {% if page == "stats_binaries" %}
+                    <a class="dropdown-toggle" data-toggle="dropdown" href="#">Stats / Binaries<span class="caret"></span></a>
+                    {% endif %}
                     <ul class="dropdown-menu">
-                        <li><a href="#">Sensors</a></li>
+                        {% if page == "stats_binaries" %}
+                        <li><a href="#"><strong>Binaries</strong></a></li>
+                        {% endif %}
                         <li><a href="#">Source IPs</a></li>
-                        <li><a href="#">Binaries</a></li>
                     </ul>
                 </li>
+                {% else %}
                 <li class="dropdown">
-                    <a class="dropdown-toggle" data-toggle="dropdown" href="#">Browse <span class="caret"></span></a>
+                    <a class="dropdown-toggle" data-toggle="dropdown" href="#">Stats <span class="caret"></span></a>
                     <ul class="dropdown-menu">
+                        <li><a href="{% url stats_binaries %}">Binaries</a>
                         <li><a href="#">Source IPs</a></li>
-                        <li><a href="#">Binaries</a></li>
                     </ul>
                 </li>
-                
+                {% endif %}
             </ul>
             <div class="nav pull-right dropdown">
                 <a class="btn dropdown-toggle" data-toggle="dropdown" href="#"><span class="icon-user"></span> {{ user.username }} <span class="caret"></span></a>
             </div>
         </div>
     </div>
-</div>
+</div>

hbwebui/templates/stats_binaries.html

+{% extends "bootstrap.html" %}
+{% block bootstrap-head %}
+
+{% load staticfiles %}
+
+<script src="{% static "js/lodash.min.js" %}"></script>
+
+
+<title>Binaries Stats</title>
+
+
+<script>
+    function Module_Stats_Binaries_Table(container) {
+        this.container = container;
+        this.table = null;
+        
+        var t = $('<table>').addClass('table').appendTo(container);
+        
+        var timeformat = d3.time.format("%Y-%m-%d %H:%M:%S");
+        function renderTimeStamp(ts) {
+            return timeformat(new Date(ts*1000));
+        }
+        
+        this.init = function(data) {            
+            this.table = t.dataTable( {
+                'bPaginate': false,
+                'bLengthChange': false,
+                'bFilter': false,
+                'bSort': true,
+                'bInfo': false,
+                'bAutoWidth': false,
+                'sDom': '',
+                //'sDom': "<'row'<'span6'l><'span6'f>r>t<'row'<'span6'i><'span6'p>>",
+                'sPaginationType': 'bootstrap',
+                
+                'aaData': data,
+                'aoColumnDefs': [
+                    { 'sTitle': 'MD5', 'mDataProp': 'md5', 'sType': 'string', 'aTargets': [0], 'fnRender': function(o, val) { return '<a href="#">'+val+'</a>'; }},
+                    { 'sTitle': '# Attacks', 'mDataProp': 'n_count', 'sType': 'numeric', 'aTargets': [1] },
+                    { 'sTitle': '# Idents', 'mDataProp': 'n_ident', 'sType': 'numeric', 'aTargets': [2] },
+                    { 'sTitle': '# Source IPs', 'mDataProp': 'n_source_ip', 'sType': 'numeric', 'aTargets': [3] },
+                    { 'sTitle': '# Target Ports', 'mDataProp': 'n_target_port', 'sType': 'numeric', 'aTargets': [4] },
+                    { 'sTitle': 'Last', 'mDataProp': 'ts_last', 'sType': null, 'aTargets': [5], 'fnRender': function(o, val) { return renderTimeStamp(val); }},
+                    { 'sTitle': 'First', 'mDataProp': 'ts_first', 'sType': null, 'aTargets': [6], 'fnRender': function(o, val) { return renderTimeStamp(val); }},
+                ],
+                'aaSorting': [
+                    [1, 'desc'],
+                ]
+            });
+        }
+    }
+    
+    function Module_Stats_Binaries_Summary(container) {
+        this.container = container;
+        
+        this.tpl = '\
+            <h3><%= n_count %> <% print(pluralize(n_ident, "Binary", "Binaries")) %></h3>\
+            <ul>\
+                <li>collected by <%= n_ident %> <% print(pluralize(n_ident, "ident")) %></li>\
+                <li>on <%= n_target_port %> target <% print(pluralize(n_target_port, "port")) %></li>\
+                <li>from <%= n_source_ip %> source <% print(pluralize(n_source_ip, "IP")) %></li>\
+                <li>between <%= ts_first %> - <%= ts_last %></li>\
+                <li>in <%= n_total %> <% print(pluralize(n_total, "attack")) %> in total</li>\
+            </ul>';
+        
+        var timeformat = d3.time.format("%Y-%m-%d %H:%M:%S");
+        function renderTimeStamp(ts) {
+            return timeformat(new Date(ts*1000));
+        }
+        
+        function pluralize(n, s, p) {
+            console.log(n.print);
+            if (n == 1)
+                return s;
+            else {
+                if (p !== undefined)
+                    return p;
+                else
+                    return s + 's';
+            }
+        }
+        
+        this.init = function(data) {
+            //_.templateSettings = {
+            //   'interpolate': /##(.+?)##/g
+            //};
+            
+            data.ts_first = renderTimeStamp(data.ts_first);
+            data.ts_last = renderTimeStamp(data.ts_last);
+            data.pluralize = pluralize;
+                        
+            container.eq(0).html(_.template(this.tpl, data));
+        }
+    }
+</script>
+
+<script>
+    var urls = {
+        'r_stats_binaries_summary': '{% url r_stats_binaries_summary %}',
+        'r_stats_binaries': '{% url r_stats_binaries %}',
+    };
+    
+    var idents = {{ idents|safe }};
+    
+    
+    $(
+        function() {
+            $.ajax(urls.r_stats_binaries_summary).done(function(data) {
+                mod_stats_binaries_summary = new Module_Stats_Binaries_Summary($('#mod-stats-binaries-summary'));
+                mod_stats_binaries_summary.init(data);
+            });
+            
+            $.ajax(urls.r_stats_binaries).done(function(data) {
+                mod_stats_binaries_table = new Module_Stats_Binaries_Table($('#mod-stats-binaries-table'));
+                mod_stats_binaries_table.init(data);
+            });
+        }
+    );
+</script>
+
+{% endblock %}
+
+{% block bootstrap-body %}
+{% include "navbar.html" %}
+
+<div class="container-fluid">
+    <div class="row-fluid">
+        <div id="mod-stats-binaries-summary"></div>
+        <hr/>
+    </div>
+    <div class="row-fluid">
+        <div id="mod-stats-binaries-table"></div>
+    </div>
+</div>
+
+{% endblock %}
 
 urlpatterns = patterns('hbwebui.views',
     url(r'^$', 'dashboard', name='dashboard'),
+    url(r'^stats/binaries$', 'stats_binaries', name='stats_binaries'),
+    
+    url(r'^r/stats/binaries$', 'r_stats_binaries', name='r_stats_binaries'),
+    url(r'^r/stats/binaries/summary$', 'r_stats_binaries_summary', name='r_stats_binaries_summary'),
     
     url(r'^r/myidents', 'myidents', name='myidents'),
     
         },
         name='logout'
     ),
-)
+)
         dict(zip([col[0] for col in desc], row)) for row in cursor.fetchall()
     ]
 
+def dictfetchmany(cursor, rows):
+    desc = cursor.description
+    return [
+        dict(zip([col[0] for col in desc], row)) for row in cursor.fetchmany(rows)
+    ]
+
 def get_idents(user):
     values_list = IdentToGroupMapping.objects.filter(group=user.groups.all()).values_list('ident')
     
     idents = [ t[0] for t in values_list ]
     
-    return idents;
+    return idents;
 
 from django.db import connections
 from django.core import serializers
-import json, csv
+import json, csv, calendar
 from django.contrib import auth
 from models import Ident, IdentToGroupMapping
 import util
 
+import queries
+
 @login_required(login_url='login')
 def dashboard(request):    
     idents = request.session['idents']
                 'idents': json.dumps(idents),
         })
         return HttpResponse(t.render(c))
+        
+def stats_binaries(request):    
+    idents = request.session['idents']
+    
+    if not idents:
+        t = loader.get_template('error_noidents.html')
+        c = RequestContext(request, {
+                
+        })
+        return HttpResponse(t.render(c));
+    else:
+        q = queries.Stats_Binaries()
+        q.filter_ident(idents)
+        q.order_n_count(desc=True)
+        cursor = q.execute()
+        qr = util.dictfetchall(cursor)
+        
+        r = []
+        for d in qr:
+            o = {}
+            o['md5'] = d['md5']
+            o['n_count'] = d['n_count']
+            o['n_ident'] = d['n_ident']
+            o['n_source_ip'] = d['n_source_ip']
+            o['n_target_port'] = d['n_target_port']
+            o['ts_last'] = calendar.timegm(d['ts_last'].utctimetuple())
+            o['ts_first'] = calendar.timegm(d['ts_first'].utctimetuple())
+            r.append(o)
+        
+        t = loader.get_template('stats_binaries.html')
+        c = RequestContext(request, {
+                'page': 'stats_binaries',
+                'idents': json.dumps(idents),
+                'r': json.dumps(r)
+        })
+        return HttpResponse(t.render(c))
+
+def r_stats_binaries(request):
+    if not request.user.is_authenticated(): return HttpResponseForbidden()
+    
+    idents = request.session['idents']
+    if not idents:
+        return HttpResponseNotFound('%s: no mapping' % request.user)
+    
+    
+    q = queries.Stats_Binaries()
+    
+    q.filter_ident(idents)
+    q.order_n_count(desc=True)
+    
+    cursor = q.execute()
+    qr = util.dictfetchall(cursor)
+    
+    r = []
+    for d in qr:
+        o = {}
+        o['md5'] = d['md5']
+        o['n_count'] = d['n_count']
+        o['n_ident'] = d['n_ident']
+        o['n_source_ip'] = d['n_source_ip']
+        o['n_target_port'] = d['n_target_port']
+        o['ts_last'] = calendar.timegm(d['ts_last'].utctimetuple())
+        o['ts_first'] = calendar.timegm(d['ts_first'].utctimetuple())
+        r.append(o)
+        
+    response = HttpResponse(mimetype='application/json')
+    json.dump(r, response)
+    
+    return response
+
+def r_stats_binaries_summary(request):
+    if not request.user.is_authenticated(): return HttpResponseForbidden()
+    
+    idents = request.session['idents']
+    if not idents:
+        return HttpResponseNotFound('%s: no mapping' % request.user)
+    
+    
+    q = queries.Stats_Binaries_Summary()
+    
+    q.filter_ident(idents)
+    
+    cursor = q.execute()
+    qr = util.dictfetchall(cursor)
+    
+    r = {}
+    r['n_count'] = qr[0]['n_count']
+    r['n_ident'] = qr[0]['n_ident']
+    r['n_source_ip'] = qr[0]['n_source_ip']
+    r['n_target_port'] = qr[0]['n_target_port']
+    r['ts_last'] = calendar.timegm(qr[0]['ts_last'].utctimetuple())
+    r['ts_first'] = calendar.timegm(qr[0]['ts_first'].utctimetuple())
+    r['n_total'] = qr[0]['n_total']
+    
+    response = HttpResponse(mimetype='application/json')
+    json.dump(r, response)
+    
+    return response
 
 # example
 def myidents(request):
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.