Commits

Martin von Löwis  committed b805071

Compress simple page to reduce size of object.

  • Participants
  • Parent commits 90fe335

Comments (0)

Files changed (2)

 import httplib, xmlrpclib, time, pickle, urllib2, binascii, os, logging, re
+import cStringIO, gzip, zlib
 try:
     from xml.etree.cElementTree import *
 except ImportError:
     else:
         h.putrequest('GET', '/simple/')
         obj = m
+    h.putheader('Accept-Encoding', 'gzip')
     h.putheader('User-Agent', UA)
     h.endheaders()
     try:
     except DownloadError, e:
         raise StepFailed(e)
     html = r.read()
+    if r.getheader('Content-Encoding') == 'gzip':
+        html = gzip.GzipFile(fileobj=cStringIO.StringIO(html), mode='r').read()
     if r.status == 404:
         if project:
             delete_package(obj)
     if r.status != 200:
         raise ValueError, "Status %d on %s" % (r.status, project)
     if obj:
-        obj.simple = html
+        obj.simple = zlib.compress(html)
     else:
         obj = model.Project(key_name=uproject, simple=html)
     if project:
-import os, logging, time
+import os, logging, time, zlib
 from google.appengine.ext import webapp, blobstore
 from google.appengine.ext.webapp import template, blobstore_handlers
 import model, fetch, stats
         path = path.rstrip('/')
         if not path:
             p = model.MirrorState.all().fetch(1)[0]
+            html = zlib.decompress(p.simple)
         elif '/' in path:
             return self.error(404)
         else:
                 self.response.set_status(404)
                 self.response.out.write("<html><head><title>404 Not Found</title></head><body>Package %s does not exist</body></html>" % path)
                 return
-        self.response.out.write(p.simple)
+            html = p.simple
+        self.response.out.write(html)
 
 class LastModified(webapp.RequestHandler):
     def get(self):