Commits

Ralph Bean committed 48ab0f3

pep 8 compliance.

Comments (0)

Files changed (8)

pep381client/__init__.py

 from __future__ import with_statement
-import cPickle, os, xmlrpclib, time, urllib2, httplib, socket
+
+import cPickle
+import os
+import xmlrpclib
+import time
+import urllib2
+import httplib
+import socket
+
 from xml.etree import ElementTree
 import xml.parsers.expat
 import sqlite
 
 # library config
 pypi = 'pypi.python.org'
-BASE = 'http://'+pypi
+BASE = 'http://' + pypi
 SIMPLE = BASE + '/simple/'
 version = '1.5'
-UA = 'pep381client/'+version
+UA = 'pep381client/' + version
 
 # Helpers
 
 _proxy = None
+
+
 def xmlrpc():
     global _proxy
     if _proxy is None:
-        _proxy = xmlrpclib.ServerProxy(BASE+'/pypi')
+        _proxy = xmlrpclib.ServerProxy(BASE + '/pypi')
         _proxy.useragent = UA
     return _proxy
 
 _conn = None
+
+
 def http():
     global _conn
     if _conn is None:
         _conn.connect()
     return _conn
 
+
 def now():
     return int(time.time())
 
-# Main class
 
 class Synchronization:
     "Picklable status of a mirror"
     def __init__(self):
         self.homedir = None
         self.quiet = False
-        
+
         # time stamps: seconds since 1970
-        self.last_completed = 0 # when did the last run complete
+        self.last_completed = 0  # when did the last run complete
         self.last_started = 0   # when did the current run start
 
         self.complete_projects = set()
     @staticmethod
     def load(homedir, storage=None):
         res = cPickle.load(open(os.path.join(homedir, "status"), "rb"))
-        res.storage = storage or sqlite.SqliteStorage(os.path.join(homedir, "files"))
+        res.storage = storage or sqlite.SqliteStorage(
+            os.path.join(homedir, "files")
+        )
         res.defaults()
-        res.homedir = homedir # override pickled value in case it got relocated
+        # override pickled value in case it got relocated
+        res.homedir = homedir
         return res
 
     #################### Synchronization logic ##############################
         status.homedir = targetdir
         status.last_started = now()
         status.projects_to_do = set(xmlrpc().list_packages())
-        status.storage = storage or sqlite.SqliteStorage(os.path.join(status.homedir, "files"))
+        status.storage = storage or sqlite.SqliteStorage(
+            os.path.join(status.homedir, "files")
+        )
         status.store()
         return status
 
         if self.last_started == 0:
             # no synchronization in progress. Fetch changelog
             self.last_started = now()
-            changes = xmlrpc().changelog(self.last_completed-1)
+            changes = xmlrpc().changelog(self.last_completed - 1)
             if not changes:
                 self.update_timestamp(self.last_started)
                 return
                 self.maybe_copy_file(project, file)
             # files start with /; remove it
             relfiles = set(p[1:] for p in files)
-            for file in self.storage.files(project)-relfiles:
+            for file in self.storage.files(project) - relfiles:
                     self.remove_file(file)
             self.complete_projects.add(project)
             self.projects_to_do.remove(project)
         self.store()
 
     def update_timestamp(self, when):
-        with open(os.path.join(self.homedir, "web", "last-modified"), "wb") as f:
+        fname = os.path.join(self.homedir, "web", "last-modified")
+        with open(fname, "wb") as f:
             f.write(time.strftime("%Y%m%dT%H:%M:%S\n", time.gmtime(when)))
 
     def copy_simple_page(self, project):
         project = project.encode('utf-8')
         h = http()
         if project:
-             h.putrequest('GET', '/simple/'+urllib2.quote(project)+'/')
+            h.putrequest('GET', '/simple/' + urllib2.quote(project) + '/')
         else:
-             h.putrequest('GET', '/simple/')
+            h.putrequest('GET', '/simple/')
         h.putheader('User-Agent', UA)
         h.endheaders()
         r = h.getresponse()
             # package exists
             return None
         if r.status != 200:
-            raise ValueError, "Status %d on %s" % (r.status, project)
-        project_simple_dir = os.path.join(self.homedir, 'web', 'simple', project)
+            raise ValueError("Status %d on %s" % (r.status, project))
+        project_simple_dir = os.path.join(
+            self.homedir, 'web', 'simple', project
+        )
         if not os.path.exists(project_simple_dir):
             os.mkdir(project_simple_dir)
         with open(os.path.join(project_simple_dir, 'index.html'), "wb") as f:
             f.write(html)
-        h.putrequest('GET', '/serversig/'+urllib2.quote(project)+'/')
+        h.putrequest('GET', '/serversig/' + urllib2.quote(project) + '/')
         h.putheader('User-Agent', UA)
         h.endheaders()
         r = h.getresponse()
             if not project:
                 # index page is unsigned
                 return
-            raise ValueError, "Status %d on signature for %s" % (r.status, project)
-        with open(os.path.join(self.homedir, "web", "serversig", project), "wb") as f:
+            raise ValueError(
+                "Status %d on signature for %s" % (r.status, project)
+            )
+        fname = os.path.join(self.homedir, "web", "serversig", project)
+        with open(fname, "wb") as f:
             f.write(sig)
         return html
 
             path = path[1:]
         lpath = os.path.join(self.homedir, "web", path)
         if r.status == 200:
-            self.storage.remove_file(path) # readd when done downloading
+            self.storage.remove_file(path)  # readd when done downloading
             data = r.read()
             dirname = os.path.dirname(lpath)
             if not os.path.exists(dirname):
     def delete_project(self, project):
         for f in self.storage.files(project):
             self.remove_file(f)
-        project_simple_dir = os.path.join(self.homedir, "web", "simple", project)
+        project_simple_dir = os.path.join(
+            self.homedir, "web", "simple", project
+        )
         if os.path.exists(project_simple_dir):
             index_file = os.path.join(project_simple_dir, "index.html")
             if os.path.exists(index_file):
                 os.unlink(index_file)
             os.rmdir(project_simple_dir)
-        project_serversig_dir = os.path.join(self.homedir, "web", "serversig", project)
+        project_serversig_dir = os.path.join(
+            self.homedir, "web", "serversig", project
+        )
         if os.path.exists(project_serversig_dir):
             os.unlink(project_serversig_dir)
         if project in self.projects_to_do:

pep381client/apache_reader.py

 import os
 
 # list of recognized user agents
-SETUPTOOLS_UA = (re.compile((r'^.* setuptools/(?P<version>[0-9]\..*)$')), 'setuptools/%s')
-URLLIB_UA = (re.compile(r'^Python-urllib/(?P<version>[23]\.[0-9])$'), 'Python-urllib/%s')
-SAFARI_UA = (re.compile(r'^Mozilla.* .* Version/(?P<version>.*) Safari/.*$'), 'Safari/%s')
-GOOGLEBOT = (re.compile(r'Googlebot-Mobile/(?P<version>.*);'), 'Googlebot-Mobile/%s')
-MSNBOT = (re.compile(r'^msnbot/(?P<version>.*) '), 'msnbot/%s')
-FIREFOX_UA = (re.compile(r'^Mozilla.*? Firefox/(?P<version>[23])\..*$'), 'Firefox/%s')
-PLAIN_MOZILLA = (re.compile(r'^Mozilla/(?P<version>.*?) '), 'Mozilla/%s')
+SETUPTOOLS_UA = (
+    re.compile((r'^.* setuptools/(?P<version>[0-9]\..*)$')),
+    'setuptools/%s'
+)
+URLLIB_UA = (
+    re.compile(r'^Python-urllib/(?P<version>[23]\.[0-9])$'),
+    'Python-urllib/%s'
+)
+SAFARI_UA = (
+    re.compile(r'^Mozilla.* .* Version/(?P<version>.*) Safari/.*$'),
+    'Safari/%s'
+)
+GOOGLEBOT = (
+    re.compile(r'Googlebot-Mobile/(?P<version>.*);'),
+    'Googlebot-Mobile/%s'
+)
+MSNBOT = (
+    re.compile(r'^msnbot/(?P<version>.*) '),
+    'msnbot/%s'
+)
+FIREFOX_UA = (
+    re.compile(r'^Mozilla.*? Firefox/(?P<version>[23])\..*$'),
+    'Firefox/%s'
+)
+PLAIN_MOZILLA = (
+    re.compile(r'^Mozilla/(?P<version>.*?) '),
+    'Mozilla/%s'
+)
 
-logre = re.compile(r"\[(?P<day>..)/(?P<month>...)/(?P<year>....):"
-                   r"(?P<hour>..):(?P<min>..):(?P<sec>..) "
-                   r'(?P<zone>.*)\] "GET (?P<path>[^ "]+) HTTP/1.." 200 .*? (?:".*?")? '
-                   r'"(User-Agent: )?(?P<useragent>.*)"$', re.DOTALL)
+logre = re.compile(
+    r"\[(?P<day>..)/(?P<month>...)/(?P<year>....):"
+    r"(?P<hour>..):(?P<min>..):(?P<sec>..) "
+    r'(?P<zone>.*)\] "GET (?P<path>[^ "]+) HTTP/1.." 200 .*? (?:".*?")? '
+    r'"(User-Agent: )?(?P<useragent>.*)"$', re.DOTALL)
 
-month_names=['jan','feb','mar','apr','may','jun',
-             'jul','aug','sep','oct','nov','dec']
+month_names = ['jan', 'feb', 'mar', 'apr', 'may', 'jun',
+               'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
 month_index = {}
 
 for i in range(12):
-    month_index[month_names[i]] = i+1
+    month_index[month_names[i]] = i + 1
+
 
 def month_to_index(month):
     return month_index[month.lower()]
 
+
 class ApacheLogReader(object):
     """provides an iterator over apache logs"""
 
         return path[-2]
 
     def get_simplified_ua(self, user_agent):
-        """returns a simplified version of the user agent""" 
-        for expr, repl in (URLLIB_UA, SETUPTOOLS_UA, SAFARI_UA, GOOGLEBOT, 
+        """returns a simplified version of the user agent"""
+        for expr, repl in (URLLIB_UA, SETUPTOOLS_UA, SAFARI_UA, GOOGLEBOT,
                            MSNBOT, FIREFOX_UA, PLAIN_MOZILLA):
             res = expr.search(user_agent)
             if res is not None:
     def next(self):
 
         while True:
-            line = self._data.next().strip() 
+            line = self._data.next().strip()
             m = logre.search(line)
             if m is None:
                 continue
             return res
 
         raise StopIteration
-

pep381client/apache_stats.py

 import os
-import csv 
+import csv
 import bz2
 import gzip
 import re
 
 from apache_reader import ApacheLogReader
 
+
 class LocalStats(object):
     """Base class that writes the log file
     """
         elif compression == 'gz':
             return gzip.open(path, mode)
         return open(path, mode)
-    
-    def _build_stats(self, logfile, fileobj, files_url='/packages', 
+
+    def _build_stats(self, logfile, fileobj, files_url='/packages',
                      filter=None, compression=None):
         """Builds a stats file
-        
+
         - logfile: path to the original log file, or callable
         - fileobj : a file object or a path to create a file
-        - files_url : a filter that define the beginnin of package urls 
-        - filter: if given, a callable that receives the 
-        current line. if the callable returns True, 
+        - files_url : a filter that define the beginnin of package urls
+        - filter: if given, a callable that receives the
+        current line. if the callable returns True,
         the line is not included
         """
         if isinstance(fileobj, str):
                 if filter(log):
                     continue
             filename = log['filename']
-            user_agent = log['useragent'] 
+            user_agent = log['useragent']
             package_name = log['packagename']
             key = (filename, user_agent, package_name)
             count = log.get('count', 1)
     def build_daily_stats(self, year, month, day, logfile, fileobj,
                           files_url='/packages', compression=None):
         """creates a daily stats file using an apache log file.
-        
-        - year, month, day: values for the day 
+
+        - year, month, day: values for the day
         - logfile : path to the log file, or callable
         - fileobj : a file object or a path to create a file
         - files_url : a filter that define the beginning of package urls
         """
         def _filter(log):
-            return (day != log['day'] or month != log['month'] or 
+            return (day != log['day'] or month != log['month'] or
                     year != log['year'])
 
         self._build_stats(logfile, fileobj, files_url, _filter, compression)
 
-
     def build_monthly_stats(self, year, month, logfile, fileobj,
                             files_url='/packages', compression=None):
         """creates a monthly stats file using an apache log file.
-        
+
         - year, month: values for the month
         - logfile : path to the log file
         - fileobj : a file object or a path to create a file
         if directory is not None:
             filename = os.path.join(directory, filename)
 
-        self.build_daily_stats(year, month, day, logfile, filename, 
+        self.build_daily_stats(year, month, day, logfile, filename,
                                compression='bz2')
 
+
 class ApacheLocalStats(LocalStats):
     """concrete class that uses the ApacheLogReader"""
     def _get_logs(self, logfile, files_url):
         return ApacheLogReader(logfile, files_url)
 
+
 class ApacheDistantLocalStats(ApacheLocalStats):
     """Concrete class that gets the data from a distant file"""
     is_url = re.compile(r'^http://')
         self.timeout = timeout
 
     def get_and_cache(self, url):
-        """retrieve the distant file and add it in the local 
+        """retrieve the distant file and add it in the local
         cache"""
         basename = url.split('/')[-1]
         filename = os.path.join(self.cache_folder, basename)
         if os.path.exists(filename):
             # in cache, let's return it
             return filename, open(filename)
-        
+
         # not in cache, we need to retrieve it
         # and store it
         oldtimeout = socket.getdefaulttimeout()
             f.write(content)
         finally:
             f.close()
-        
+
         return filename, open(filename)
 
-    def read_stats(self, stats_file):  
+    def read_stats(self, stats_file):
         """retrieve a distant file and works with it"""
-        if self.is_url.search(stats_file) is not None: 
+        if self.is_url.search(stats_file) is not None:
             path, fileobj = self.get_and_cache(stats_file)
             if path == '':
                 return iter([])
         return ApacheLocalStats.read_stats(self, path)
-

pep381client/sqlite.py

-import sqlite3, os
+import sqlite3
+import os
+
 
 class SqliteStorage(object):
 
 
     def files(self, project):
         cursor = self.conn.cursor()
-        cursor.execute("select filename from files where project=?", (project,))
+        cursor.execute(
+            "select filename from files where project=?", (project,)
+        )
         return set(r[0] for r in cursor.fetchall())
 
     def etag(self, filename):
         cursor = self.conn.cursor()
-        cursor.execute("select etag from files where filename=?", (filename,))
+        cursor.execute(
+            "select etag from files where filename=?", (filename,)
+        )
         res = cursor.fetchone()
         if res:
             return res[0]
 
     def add_file(self, project, filename, etag):
         cursor = self.conn.cursor()
-        cursor.execute("insert into files(project, filename, etag) values(?, ?, ?)",
-                       (project, filename, etag))
+        cursor.execute(
+            "insert into files(project, filename, etag) values(?, ?, ?)",
+            (project, filename, etag)
+        )
 
     def remove_file(self, filename):
         cursor = self.conn.cursor()

scripts/pep381checkfiles

 #!/usr/bin/env python
-import sys, os, shutil, optparse, hashlib, time
+
+import sys
+import os
+import shutil
+import optparse
+import hashlib
+import time
+
 from xml.etree import ElementTree
+
 try:
     import pep381client
 except ImportError:
     # See whether we are run out of a source tree
-    module_path = os.path.join(os.path.dirname(sys.argv[0]), 
+    module_path = os.path.join(os.path.dirname(sys.argv[0]),
                                '..')
     if os.path.exists(module_path):
         sys.path.append(module_path)
             incomplete_packages.add(package)
             print "Missing file", fn
             continue
-        if "md5="+hashlib.md5(open(fn,'rb').read()).hexdigest() != md5:
+        if "md5=" + hashlib.md5(open(fn, 'rb').read()).hexdigest() != md5:
             print "Bad md5", fn
             continue
 
         # Reload pickled state
         s = pep381client.Synchronization.load(targetdir)
         s.projects_to_do.update(incomplete_packages)
-        # claim that a synchronization was aborted and 
+        # claim that a synchronization was aborted and
         # needs to be restarted
         s.last_started = s.last_completed
         s.store()

scripts/pep381run

 #!/usr/bin/env python
-import sys, os, shutil, optparse
+
+import sys
+import os
+import shutil
+import optparse
+
 try:
     import pep381client
 except ImportError:
     # See whether we are run out of a source tree
-    module_path = os.path.join(os.path.dirname(sys.argv[0]), 
+    module_path = os.path.join(os.path.dirname(sys.argv[0]),
                                '..')
     if os.path.exists(module_path):
         sys.path.append(module_path)
     import pep381client
 
 opts = optparse.OptionParser(usage="Usage: pep381run [OPTIONS] <targetdir>")
-opts.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False,
-        help="Don't print status information")
+opts.add_option(
+    "-q", "--quiet", dest="quiet", action="store_true", default=False,
+    help="Don't print status information")
 options, args = opts.parse_args()
 
 if len(args) != 1:

scripts/pep381sync.cgi

 #!/usr/bin/env python
-# replace this with actual mirror location, 
+
+# replace this with actual mirror location,
 # or set PYPITARGET environment variable
 targetdir = '/pypi'
 
-import os, sys, signal
+import os
+import sys
 
 dirname = os.path.dirname(sys.argv[0])
 

scripts/processlogs

 #!/usr/bin/python
-# Generates download stats for all days in the given log files,
-# except for the oldest and the newest day.
-import sys, os, csv
+""" Generates download stats for all days in the given log files,
+except for the oldest and the newest day.
+"""
+
+import sys
+import os
+
 # See whether we are run out of a source tree
 module_path = os.path.dirname(os.path.dirname(sys.argv[0]))
 if os.path.exists(os.path.join(module_path, "pep381client")):
     sys.path.append(module_path)
 from pep381client import apache_reader, apache_stats
 
+
 def usage(msg=None):
     if msg:
         print msg
 
 days = sorted(days)[1:-1]
 
+
 class Stats(apache_stats.LocalStats):
     def _get_logs(self, logfile, files_url):
         return records
+
 stats = Stats()
-for year,month,day in days:
-    stats.build_local_stats(year, month, day, None, os.path.join(statsdir, 'days'))
+
+for year, month, day in days:
+    stats.build_local_stats(year, month, day, None,
+                            os.path.join(statsdir, 'days'))