Commits

Luke Plant committed 61e8785

Initial import

Comments (0)

Files changed (15)

+Luke Plant <L.Plant.98@cantab.net>
+
+With thanks for contributions from:
+
+  'nesh' <nesh _at_ studioquattro _dot_ co _dot_ yu>
+Changelog
+=========
+
+Version 1.5 - 2010-12-01
+------------------------
+
+* Re-branded as django-output-validator and packaged properly.
+
+  If you used the previous version, you should drop the old
+  'validator_validationfailure' table (assuming it doesn't have any data you
+  need, of course). Then go through the installation instructions in the README
+  and update the name/values of the relevant settings.
+
+
+Version 1.4 - 2008-04-28
+------------------------
+
+* Changed maxlength to max_length, as per change in Django.
+* Corrections to instructions (thanks to Gary Wilson)
+* Fixed deprecation warnings (thanks to Gary Wilson)
+
+
+Version 1.3 - 2007-11-05
+------------------------
+
+* Updated for unicodisation of Django.
+
+  This is a BACKWARDS INCOMPATIBLE change.
+
+  The problem was caused by the fact that you used to able to store arbitrary
+  binary data in a TextField, which is no longer possible. As a result, I am
+  using base64 encoding for any pickled objects. I haven't written an upgrade
+  script for the database (since I personally keep the list of failed pages to
+  zero). If you are upgrading from a previous version, any of your existing
+  ValidationFailure objects will be corrupted (the 'request' and 'response' data
+  will be lost). Either deal with the errors before upgrading, or write a
+  conversion script of some kind :-)
+
+Version 1.2 - 2007-04-18
+------------------------
+
+* Fixed bug that occurred when settings.VALIDATOR_APP_IGNORE_PATHS wasn't set
+* Added logic to stop duplicate failures being logged
+
+Version 1.1 - 2005-12-14
+------------------------
+
+* Added optional VALIDATOR_APP_IGNORE_PATHS setting.
+* Added support for mod_python handler - thanks to 'nesh'.
+* Added a setup.py script.
+
+Version 1.0 - 2005-11-19
+------------------------
+* Initial release
+recursive-include . *.html
+include AUTHORS
+include *.rst
+=======================
+Django output validator
+=======================
+
+This app validates all the HTML pages (or other data) that is generated by your
+Django project. This is meant to be used only in development.
+
+Installation
+============
+
+* Run setup.py to install the package into your python path.
+
+* Add "output_validator" to your INSTALLED_APPS setting.
+
+* If you have removed ``"django.template.loaders.app_directories.Loader"`` from
+  your TEMPLATE_LOADERS, you need to add the 'templates' folder to your
+  TEMPLATE_DIRS setting.
+
+* Insert the middleware
+  ``"output_validator.middleware.ValidatorMiddleware"``
+  near the beginning of the middleware list (which means it will get
+  the the response object after everything else). It must be after
+  every middleware that does post-processing, but mustn't be after
+  GZip, since it can't handle gzipped HTML. ( I just disable the GZip
+  middleware for development)
+
+* Alter your URL conf to include the URLs for the validator. You need
+  this line inserted somewhere::
+
+      (r'^validator/', include('output_validator.urls'))
+
+* Add a setting to tell the app where to find the 'validate'
+  executable used for validation. This is a dictionary of mimetypes
+  and corresponding validators, allowing this app to be extended to
+  any other generated content::
+
+      OUTPUT_VALIDATOR_VALIDATORS = {
+        'text/html': '/usr/bin/validate',
+        'application/xml+xhtml': '/usr/bin/validate',
+      }
+
+  I usually use a small wrapper for this executable that pops up
+  a message when it fails - the following works for GNOME
+  (if you have the notify-send program installed)::
+
+      #!/bin/sh
+      validate "$1" || {
+          notify-send "Validation failed";
+      }
+
+* Finally, run the django admin script to set up the database tables::
+
+    ./manage.py --settings="yourproject.settings" syncdb
+
+  OR, if you are using South::
+
+    ./manage.py --settings="yourproject.settings" migrate output_validator
+
+* Optionally, set the following settings:
+
+  * OUTPUT_VALIDATOR_IGNORE_PATHS - this is a list of path prefixes that
+    will be ignored.  For example, if you have the admin at ``/admin/``
+    you can ignore any errors in the admin with this::
+
+        OUTPUT_VALIDATOR_IGNORE_PATHS = [
+            '/admin/',
+        ]
+
+
+Usage
+=====
+
+When browsing any of your pages in development, all HTML will be validated. If
+it fails, it will be logged. You can see all failures at
+'http://localhost:8000/validator/' (assuming local development and the URL conf
+suggested above). Use the app to delete old failures once they have been fixed.

output_validator/__init__.py

+

output_validator/middleware.py

+from output_validator.models import ValidationFailure
+
+
+class ValidatorMiddleware(object):
+    def process_response(self, request, response):
+        if response.status_code == 200:
+            ValidationFailure.do_validation(request, response)
+        return response

output_validator/migrations/0001_initial.py

+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'ValidationFailure'
+        db.create_table('output_validator_validationfailure', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('timestamp', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
+            ('path', self.gf('django.db.models.fields.TextField')()),
+            ('method', self.gf('django.db.models.fields.CharField')(max_length=6)),
+            ('request', self.gf('django.db.models.fields.TextField')(default='', blank=True)),
+            ('response', self.gf('django.db.models.fields.TextField')(default='', blank=True)),
+            ('errors', self.gf('django.db.models.fields.TextField')()),
+        ))
+        db.send_create_signal('output_validator', ['ValidationFailure'])
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'ValidationFailure'
+        db.delete_table('output_validator_validationfailure')
+
+
+    models = {
+        'output_validator.validationfailure': {
+            'Meta': {'ordering': "('-timestamp',)", 'object_name': 'ValidationFailure'},
+            'errors': ('django.db.models.fields.TextField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'method': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
+            'path': ('django.db.models.fields.TextField', [], {}),
+            'request': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+            'response': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+            'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'})
+        }
+    }
+
+    complete_apps = ['output_validator']

output_validator/migrations/__init__.py

Empty file added.

output_validator/models.py

+import base64
+import cPickle
+import copy
+import datetime
+import os
+import tempfile
+
+from django.core.handlers.modpython import ModPythonRequest
+from django.db import models
+
+
+class ValidationFailure(models.Model):
+    timestamp = models.DateTimeField("Time", default=datetime.datetime.now)
+    path = models.TextField("Request path")
+    method = models.CharField("HTTP method", max_length=6)
+    request = models.TextField("Request object", default='', blank=True)
+    response = models.TextField("Response object", default='', blank=True)
+    errors = models.TextField("Errors")
+
+    def __repr__(self):
+        return self.method + " " + self.path
+
+    def get_request_formatted(self):
+        import cPickle
+        try:
+            return repr(cPickle.loads(base64.decodestring(self.request)))
+        except EOFError, UnpicklingError:
+            return None
+
+    def get_response(self):
+        import cPickle
+        try:
+            return cPickle.loads(base64.decodestring(self.response))
+        except EOFError, UnpicklingError:
+            return None
+
+    class Meta:
+        ordering = ('-timestamp',)
+
+    class Admin:
+        fields =  (
+            (None, {'fields': ('timestamp', 'path', 'method', 'errors')}),
+        )
+
+
+    def do_validation(request, response):
+        """
+        Do validation on response and log if it fails.
+        """
+        from django.conf import settings
+        try:
+            OUTPUT_VALIDATOR_IGNORE_PATHS = settings.OUTPUT_VALIDATOR_IGNORE_PATHS
+        except AttributeError:
+            OUTPUT_VALIDATOR_IGNORE_PATHS = ()
+
+
+        try:
+            content_type = response['Content-Type'].split(';')[0]
+            validator = settings.OUTPUT_VALIDATOR_VALIDATORS[content_type]
+        except KeyError, IndexError:
+            # no content type, or no validator for that content type
+            return
+
+        for ignore_path in OUTPUT_VALIDATOR_IGNORE_PATHS:
+            if request.path.startswith(ignore_path):
+                return
+
+        # first store data in temporary file
+        (tmpfilehandle, tmpfilepath) = tempfile.mkstemp()
+        os.write(tmpfilehandle, response.content)
+        os.close(tmpfilehandle)
+
+        # Now execute validator and get result
+        (child_stdin, child_output) = os.popen4(validator + ' ' + tmpfilepath)
+        errors = child_output.read()
+
+        # Normalise output so that we can eliminate duplicate errors
+        errors = errors.replace(tmpfilepath, '[tmpfilepath]')
+
+        # clean up
+        child_stdin.close()
+        child_output.close()
+        os.unlink(tmpfilepath)
+
+        # Only save if there was an error, and there isn't already
+        # a failure saved at the same URL with identical errors.
+        # (this isn't perfectly watertight -- you could by chance be
+        # generating identical errors with different query strings or
+        # POST data, but it's unlikely).
+
+        if len(errors) > 0 and \
+               ValidationFailure.objects.filter(errors=errors,
+                                                path=request.path).count() == 0:
+            failure = ValidationFailure(errors=errors)
+            failure.path = request.path
+            qs = request.META.get('QUERY_STRING','')
+            if qs is not None and len(qs) > 0:
+                failure.path += '?' + qs
+            failure.errors = errors
+
+            if isinstance(request, ModPythonRequest):
+                # prepopulate vars
+                request._get_get()
+                request._get_post()
+                request._get_cookies()
+                request._get_files()
+                request._get_meta()
+                request._get_request()
+                request._get_raw_post_data()
+                u = request.user
+                mp = request._req
+                del request._req # get rid of mp_request
+                try:
+                    req = copy.deepcopy(request)
+                except Exception, e:
+                    req = "Couldn't stash a copy of the request: %s" % str(e)
+                request._req = mp # restore mp_request
+            else:
+                try:
+                    req = copy.deepcopy(request)
+                    # remove the stuff we can't serialize
+                    del req.META['wsgi.errors']
+                    del req.META['wsgi.file_wrapper']
+                    del req.META['wsgi.input']
+                except Exception, e:
+                    # TODO - work out why this happens
+                    req = "Couldn't stash a copy of the request: %s" % str(e)
+
+            failure.request = base64.encodestring(cPickle.dumps(req))
+            failure.response = base64.encodestring(cPickle.dumps(response))
+            failure.method = request.META['REQUEST_METHOD']
+            failure.save()
+    do_validation = staticmethod(do_validation)

output_validator/templates/output_validator/base.html

+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+<title>Validation status</title>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<style type="text/css">
+
+body {
+	margin: 0px 20px;
+	padding: 0px;
+        background-color: white;
+        color: black;
+        font-family: sans;
+}
+
+
+h1
+{
+	text-align: center;
+	font-size: 1.5em;
+	margin: 0px -20px;
+	padding: 10px 0px;
+	background-color: #e8e8ff;
+	border-bottom: 2px solid #8080a0;
+}
+
+h2
+{
+	margin-left: -10px;
+	font-size: 1.2em;
+	color: #000060;
+}
+
+h3
+{
+	margin-left: -5px;
+	font-size: 1.1em;
+}
+
+td, th, table
+{
+	border-collapse: collapse;
+	border: 1px solid #8080a0;
+	padding: 2px;
+}
+
+tr
+{
+	background-color: #e8e8ff;
+}
+
+tr.alt, th
+{
+	background-color: #ffffff;
+}
+
+th, td
+{
+	vertical-align: top;
+}
+
+pre
+{
+	margin: 0px;
+}
+
+</style>
+</head>
+<body>
+<h1>Django project validation monitor</h1>
+<div id="content">
+{% block content %}
+{% endblock %}
+</div>
+</body>
+</html>

output_validator/templates/output_validator/validationfailure_detail.html

+{% extends "output_validator/base.html" %}
+{% block content %}
+<a href="../">[up]</a>
+<h2>Validation failure details</h2>
+
+<form method="post" action="delete/">{% csrf_token %}<div>
+<input type="submit" name="delete" value="Delete" />
+</div>
+</form>
+
+<table width="100%">
+	<tr>
+		<th scope="row">Time</th>
+		<td>{{ object.timestamp|date:"d M y h:i" }}</td>
+	</tr>
+	<tr>
+		<th scope="row">Request</th>
+		<td>{{ object.method }} {{ object.path }}
+		{% if object.method == 'GET' %}<a href="{{ object.path|escape }}">[go]</a>{% endif %}
+		</td>
+	</tr>
+	<tr>
+		<th scope="row">Errors</th>
+		<td><div><pre>{{ object.errors }}</pre></div></td>
+	</tr>
+	<tr>
+		<th scope="row">Response content</th>
+		<td><div><pre>{{ object.get_response.content|linenumbers }}</pre></div></td>
+	</tr>
+	<tr>
+		<th scope="row">Original request</th>
+		<td><div class="python">{{ object.get_request_formatted }}</div></td>
+	</tr>
+</table>
+{% endblock %}

output_validator/templates/output_validator/validationfailure_list.html

+{% extends "output_validator/base.html" %}
+{% block content %}
+
+{% if object_list %}
+	<form method="post" action="delete/">{% csrf_token %}
+	<div class="errorsfound"><p>{{ object_list|length }} page(s) with errors</p></div>
+	<div>
+	<table>
+		<tr class="header">
+			<th scope="col">Time</th>
+			<th scope="col">Request</th>
+			<th scope="col">Details</th>
+			<th scope="col">Delete</th>
+		</tr>
+		{% for failure in object_list %}
+		<tr {% if forloop.counter|divisibleby:"2" %}class="alt"{% endif %}>
+			<td>{{ failure.timestamp|date:"d M Y h:i" }}</td>
+			<td><span class="method">{{ failure.method }}</span> <span class="path">{{ failure.path|escape }}</span></td>
+			<td><a href="{{ failure.id }}/">Details</a></td>
+			<td><input type="checkbox" name="deleteitem{{ failure.id }}" /></td>
+		</tr>
+		{% endfor %}
+	</table>
+	<br/>
+	<input type="submit" name="deleteselected" value="Delete selected" />
+	<input type="submit" name="deleteall" value="Delete all" />
+	</div></form>
+{% else %}
+	<div class="noerrorsfound"><p>No errors found.</p></div>
+{% endif %}
+
+{% endblock %}

output_validator/urls.py

+from django.conf.urls.defaults import *
+from output_validator.models import ValidationFailure
+
+info_dict = {
+    'queryset': ValidationFailure.objects.all(),
+}
+
+urlpatterns = patterns('',
+    (r'^$',
+        'django.views.generic.list_detail.object_list',
+        dict(info_dict, allow_empty=True)),
+    (r'^(?P<object_id>\d+)/$',
+        'django.views.generic.list_detail.object_detail',
+        info_dict),
+    (r'^(?P<object_id>\d+)/delete/$',
+        'output_validator.views.delete'),
+    (r'^delete/$',
+        'output_validator.views.bulkdelete'),
+)

output_validator/views.py

+from django.http import HttpResponseRedirect
+
+from output_validator.models import ValidationFailure
+
+
+def bulkdelete(request):
+    if request.POST:
+        postkeys = request.POST.keys()
+        if 'deleteall' in postkeys:
+            ValidationFailure.objects.all().delete()
+        elif 'deleteselected' in postkeys:
+            for k in postkeys:
+                if k.startswith('deleteitem'):
+                    k = k[len('deleteitem'):]
+                    try:
+                        vf = ValidationFailure.objects.get(id=k)
+                        vf.delete()
+                    except ValidationFailure.DoesNotExist:
+                        pass
+
+    return HttpResponseRedirect("../")
+
+
+def delete(request, object_id):
+    if request.POST:
+        try:
+            vf = ValidationFailure.objects.get(id=object_id)
+            vf.delete()
+        except ValidationFailure.DoesNotExist:
+            pass
+    return HttpResponseRedirect("../../")
+
+#!/usr/bin/env python
+from setuptools import setup, find_packages
+import os
+
+
+def read(*rnames):
+    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+
+
+setup(
+    name = "django-output-validator",
+    version = '1.5',
+    packages = ['output_validator'],
+    include_package_data = True,
+
+    author = "Luke Plant",
+    author_email = "L.Plant.98@cantab.net",
+    url = "http://lukeplant.me.uk/resources/djangovalidator/",
+    description = "App to catch HTML errors (or other errors) in outgoing Django pages.",
+    long_description = (
+                        read('README.rst')
+                        + "\n\n" +
+                        read('CHANGES.rst')
+    ),
+    license = "MIT",
+    keywords = "django HTML XHTML validation validator",
+    classifiers = [
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Web Environment",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: MIT License"
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Framework :: Django",
+        "Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries",
+        "Topic :: Software Development :: Testing",
+        ]
+)