Commits

Charlie Clark committed 126e2e1

Initial port to use Google Visualistation for charts.

Comments (0)

Files changed (3)

httparchive/httparchive/__init__.py

     config.add_route('websites', '/websites/*filter')
     config.add_route('auto-complete', '/search')
     config.add_route('site', '/site/{siteid}/{run_date:.*}')
+    config.add_route('site_gviz', '/site_gviz/{siteid}/{run_date:.*}')
     config.add_route('filmstrip', '/{pageid}/filmstrip.js')
     config.add_route('harviewer', '/{wptid}/{wptrun}/harviewer.js')
     config.add_route('download', '/download/{pageid:.*}.csv')

httparchive/httparchive/views/site_gviz.py

+import datetime
+
+from site import View, Request, Page, func, Site
+
+from pyramid.view import view_config
+from gviz_data_table import Table, encoder
+from utils import format_size
+
+
+@view_config(route_name='site_gviz', renderer='templates/site_gviz.pt')
+class View(View):
+
+    @property
+    def content_type(self):
+        """Page by content type"""
+        table = Table()
+        table.add_column("label", str)
+        table.add_column("size", int)
+
+        for key, label in [('bytesHTML', 'HTML'), ('bytesImg', 'Images'),
+                           ('bytesJS', 'Scripts'), ('bytesCSS', 'Stylesheets'),
+                           ('bytesFlash', 'Flash')]:
+            value = getattr(self.summary, key, 0)
+            if value != 0:
+                value = format_size(value)
+                table.append(["{0} - {1} kB".format(label, value), value])
+        other = format_size(self.summary.bytesJSON + self.summary.bytesOther)
+        if other > 0:
+            table.append(["Other - {0} kB".format(other), other])
+        legend = "total {0} kB".format(format_size(self.summary.bytesTotal))
+
+        return encoder.encode(table)
+
+    @property
+    def response_size(self):
+        """
+        Bar chart of content types by size
+        """
+        table = Table()
+        table.add_column("Content Type", str)
+        table.add_column("Response size (kB)", int)
+        stats = self.stats
+        for k in ['GIF', 'JPG', 'PNG', 'HTML', 'JS', 'CSS']:
+            size, req = stats.get('bytes'+k, 0), stats.get('req'+k, 0)
+            if req > 0:
+                avg = float(size)/req
+                table.append([k, format_size(avg)])
+        return encoder.encode(table)
+        legend = "average response size (kB)"
+        return horizontal_bar_chart(values, labels, title, legend, color="3B356A")
+
+
+    @property
+    def image_format(self):
+        """Breakdown by image formats"""
+        total = self.stats['reqImg']
+        if total == 0:
+            return {}
+        table = Table()
+        table.add_column("Type", str)
+        table.add_column("Bytes", int)
+        values = []
+        for k in ['GIF', 'JPG', 'PNG']:
+            req = self.stats.get('req'+k, 0)
+            proportion = int(round(100 * float(req) / total))
+            table.append([k , proportion])
+            values.append(proportion)
+        other = 100 - sum(values)
+        table.append(["Other", other])
+        return encoder.encode(table)
+
+
+    @property
+    def cache_control(self):
+        """Extract and format stats"""
+
+        q = self.session.query("maxagedays", "num")
+        q = q.from_statement("""
+        select ceil( substring( resp_cache_control, (length(resp_cache_control) + 2 - locate('=ega-xam', reverse(resp_cache_control))) ) / 86400) as maxagedays,
+        count(*) as num
+        FROM requests
+        WHERE pageid = :pageid
+        AND resp_cache_control LIKE '%max-age=%'
+        group by maxagedays
+        order by maxagedays asc""").params(pageid=self.summary.pageid)
+        # additional conversion on server removed because it raises a warning which is interpreted as an error SQLAlchemy.
+        zeroOrNeg = day = month = year = yearplus = 0
+        for r in q:
+            if r.maxagedays < 1:
+                zeroOrNeg += r.num
+            elif r.maxagedays == 1:
+                day = r.num
+            elif 1 < r.maxagedays <= 30:
+                month += r.num
+            elif 30 < r.maxagedays <= 365:
+                year += r.num
+            elif r.maxagedays > 365:
+                yearplus += r.num
+        totalRequests = float(self.summary.reqTotal)
+        table = Table()
+        table.add_column("Max age", str)
+        table.add_column("Value", int)
+
+        table.append(["None", int(round(100 * (totalRequests - (zeroOrNeg + day + month + year + yearplus))/totalRequests))])
+        for k, v in (("t <= 0", zeroOrNeg),
+                     ("0 < t <= 1", day),
+                     ("1 < t <= 30", month),
+                     ("30 < t <= 365", year),
+                     ("365 < t", yearplus)
+                     ):
+            table.append([k, int(round(100 *v)/totalRequests)])
+        return encoder.encode(table)
+
+        title = "Cache-Control: max-age (days)"
+
+    @property
+    def protocols(self):
+        """Number of requests to https servers
+        """
+        q = self.session.query(Request)
+        q = q.filter(Page.pageid == self.id)
+        q = q.filter(Page.url == "https://")
+        result = float(q.count())
+        https = 100 * result / self.summary.reqTotal
+        http = 100 - https
+        table = Table()
+        table.add_column("Protocol", str)
+        table.add_column("% Requests", float)
+        table.append(["HTTP", http])
+        table.append(["HTTPS", https])
+        return encoder.encode(table)
+
+    @property
+    def trends_requests(self):
+        table = Table()
+        table.add_column("Date", datetime.date)
+        table.add_column("Total Requests", int)
+        table.add_column("Total Transfer Size", int)
+        for r in self.trends:
+            table.append([r.labelDate, int(r.reqTotal), int(r.bytesTotal)])
+        return encoder.encode(table)
+
+    @property
+    def trends(self):
+        """
+        Matches dbapi.getTrendsDataForURL except site.id is a key.
+
+        """
+        if getattr(self, "_trends"):
+            return self._trends
+
+        q = self.session.query(
+            Page.label,
+            Page.labelDate,
+            func.count(Page.label).label("numurls"),
+            Page.reqTotal,
+            Page.reqHTML,
+            Page.reqJS,
+            Page.reqCSS,
+            Page.reqImg,
+            Page.reqFlash,
+            func.round(Page.bytesTotal / 1024).label("bytesTotal"),
+            func.round(Page.bytesHTML / 1024).label("bytesHTML"),
+            func.round(Page.bytesJS / 1024).label("bytesJS"),
+            func.round(Page.bytesCSS / 1024).label("bytesCSS"),
+            func.round(Page.bytesImg / 1024).label("bytesImg"),
+            func.round(Page.bytesFlash / 1024).label("bytesFlash"),
+            Page.PageSpeed,
+            Page.numDomains
+        )
+        q = q.join(Page.site)
+        q = q.filter(Site.id == self.id)
+        q = q.group_by(Page.labelDate)
+        return q

httparchive/httparchive/views/templates/site_gviz.pt

+<metal:macro use-macro="master">
+
+<metal:slot fill-slot="style">
+<link rel="stylesheet" href="/harviewer/css/harViewer.css" type="text/css">
+</metal:slot>
+
+<metal:slot fill-slot="body">
+
+<h1><span class="protocol">http://</span> ${view.summary.url}</h1>
+
+<p class="summary" style="margin-bottom: 4px;">took ${load_time} seconds to load ${bytes} kB of data over ${requests} requests.</p>
+
+<div><a href="http://www.alexa.com/$url">Alexa rank: ${rank}</a></div>
+<div tal:define="diff view.diff"
+    tal:condition="diff">compared to ${diff['date']}:
+	<tal:repeat repeat="value diff['values']">${value['name']}
+	<span tal:attributes="class value['class']"> ${value['value']}</span>
+	</tal:repeat>
+</div>
+
+<ul class="quicklinks">
+	<li><a href="#top">Top of page</a></li>
+	<li><a href="#filmstrip">Filmstrip</a></li>
+	<li><a href="#sitestats">Stats</a></li>
+	<li><a href="#trends">Trends</a></li>
+	<li><a href="#waterfall">Waterfall</a></li>
+	<li><a href="#pagespeed">Page Speed</a></li>
+	<li><a href="#requests">Requests</a></li>
+	<li><a href="#downloads">Downloads</a></li>
+</ul>
+
+<select class="selectSite"
+onchange="document.location='./' + escape(this.options[this.selectedIndex].value)">
+	 <option tal:repeat="run view.runs" value="${run.pageid}"
+			 tal:attributes="selected pageid == run.pageid">${run.label}</option>
+</select>
+
+<section id="charts">
+<h2 id="sitestats">Stats</h2>
+
+    <script type="text/javascript" src="https://www.google.com/jsapi"></script>
+    <script type="text/javascript">
+      google.load('visualization', '1', {packages: ['corechart', 'table']});
+
+        google.setOnLoadCallback(content_type);
+    	function content_type() {
+        var data = new google.visualization.DataTable(${view.content_type});
+
+        var options = {
+          title : "Average Bytes per Page by Content Type",
+        };
+
+        var chart = new google.visualization.PieChart(document.getElementById('content_type'));
+        chart.draw(data, options);
+      }
+
+		google.setOnLoadCallback(response_size);
+    	function response_size() {
+        var data = new google.visualization.DataTable(${view.response_size});
+
+        var options = {
+          title : "Average Individual Response Size",
+        };
+
+        var chart = new google.visualization.BarChart(document.getElementById('response_size'));
+        chart.draw(data, options);
+      }
+
+		google.setOnLoadCallback(image_format);
+    	function image_format() {
+        var data = new google.visualization.DataTable(${view.image_format});
+
+        var options = {
+          title : "Image Requests by Format",
+        };
+
+        var chart = new google.visualization.PieChart(document.getElementById('image_format'));
+        chart.draw(data, options);
+      }
+
+		google.setOnLoadCallback(cache_control);
+    	function cache_control() {
+        var data = new google.visualization.DataTable(${view.cache_control});
+
+        var options = {
+          title : "Cache-Control: max-age (days)",
+        };
+
+        var chart = new google.visualization.ColumnChart(document.getElementById('cache_control'));
+        chart.draw(data, options);
+      }
+
+		google.setOnLoadCallback(protocols);
+    	function cache_control() {
+        var data = new google.visualization.DataTable(${view.protocols});
+
+        var options = {
+          title : "HTTPS Requests",
+        };
+
+        var chart = new google.visualization.PieChart(document.getElementById('protocols'));
+        chart.draw(data, options);
+      }
+
+		google.setOnLoadCallback(trend_requests);
+    	function cache_control() {
+        var data = new google.visualization.DataTable(${view.trends_requests});
+
+        var options = {
+          title : "HTTPS Requests",
+        };
+
+        var chart = new google.visualization.LineChart(document.getElementById('trends_requests'));
+        chart.draw(data, options);
+      }
+
+	</script>
+
+<a href="#bytesperpage">
+	<div id="content_type"></div>
+	<div id="table"></div>
+</a>
+
+<a href="#responsesizes">
+	<div id="response_size"></div>
+</a>
+
+<a href="#imageformats">
+	<div id="image_format"></div>
+</a>
+
+<a href="#max-age">
+    <div id="cache_control"></div>
+</a>
+
+<a href="#protocols">
+    <div id="protocols"></div>
+</a>
+
+<h2 id="trends">Trends</h2>
+
+<a href="#total_requests">
+    <div id="trends_requests"></div>
+</a>
+
+</section>
+
+</metal:slot>
+
+</metal:macro>