Source

pep381client / processlogs

#!/usr/bin/python
# Generates download stats for all days in the given log files,
# except for the oldest and the newest day.
import sys, os, csv
from pep381client import apache_reader, apache_stats

def usage(msg=None):
    if msg:
        print msg
    print "Usage: processlogs <pypi-targetdir> logfile [logfile...]"
    raise SystemExit

if len(sys.argv) < 3:
    usage()

targetdir = sys.argv[1]

if not os.path.exists(targetdir):
    usage(targetdir + ' does not exist')

if not os.path.exists(targetdir+'/web'):
    usage('Not a pypi mirror (%s/web does not exist)' % targetdir)

statsdir = targetdir+'/web/local-stats/'

if not os.path.isdir(statsdir):
    os.mkdir(statsdir)
    os.mkdir(statsdir+'days')

days = set()
records = []
for fn in sys.argv[2:]:
    for record in apache_reader.ApacheLogReader(fn, '/packages'):
        days.add((record['year'], record['month'], record['day']))
        records.append(record)

days = sorted(days)[1:-1]

class Stats(apache_stats.LocalStats):
    def _get_logs(self, logfile, files_url):
        return records
stats = Stats()
for year,month,day in days:
    stats.build_local_stats(year, month, day, None, statsdir+'days')