pypi / tools / integrate-redis-stats.py

#!/usr/bin/env python
import os
import sys
import datetime
import redis
import string

from itertools import izip, izip_longest

# Workaround current bug in docutils:
# http://permalink.gmane.org/gmane.text.docutils.devel/6324
import docutils.utils

root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path = [root] + sys.path

import store
import config

redis = redis.Redis()

# Get our search for the previous hour keys
current = datetime.datetime.utcnow()
lasthour = current - datetime.timedelta(hours=1)
search = "downloads:hour:%s:*:*" % lasthour.strftime("%y-%m-%d-%H")

# Make sure we haven't integrated this already
if redis.sismember("downloads:integrated", search):
    print("Already Integrated '%s'" % search)
    sys.exit(0)

# Fetch all of the keys
keys = redis.keys(search)

if not keys:
    print("No keys match '%s'" % search)
    sys.exit(0)

# Fetch all of the download counts (in batches of 200)
counts = []
for batch in izip_longest(*[iter(keys)] * 200):
    batch = [x for x in batch if x is not None]
    counts.extend(redis.mget(*batch))

# Combine the keys with the counts
downloads = izip(
                (int(y) for y in counts),
                (x.split(":")[-1] for x in keys),
            )

# Update the database
c = config.Config(os.path.join(root, "config.ini"))
store = store.Store(c)
cursor = store.get_cursor()
cursor.executemany(
    "UPDATE release_files SET downloads = downloads + %s WHERE filename = %s",
    (d for d in downloads if not set(d[1]) - set(string.printable)),
)
store.commit()
store.close()

# Add this to our integrated set
redis.sadd("downloads:integrated", search)
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.