Commits

ewdurbin  committed 60a2e69

allow count and worker redis URIs to be independently configured

  • Participants
  • Parent commits f9f97f5

Comments (0)

Files changed (6)

File config.ini.template

 user = pypi
 
 ; Redis
-redis_url = redis://localhost:6379/0
+queue_redis_url = redis://localhost:6379/0
+count_redis_url = redis://localhost:6379/1
 
 ; Storage Directories
 files_dir = /MacDev/svn.python.org/pypi-pep345/files
         self.fromaddr = c.get('logging', 'fromaddr')
         self.toaddrs = c.get('logging', 'toaddrs').split(',')
 
-        self.redis_url = c.get('database', 'redis_url')
+        self.queue_redis_url = c.get('database', 'queue_redis_url')
+        self.count_redis_url = c.get('database', 'count_redis_url')
 
         self.sentry_dsn = c.get('sentry', 'dsn')
 
             self.can_lock = True
 
         self.queue = queue
-        self.redis = redis
+        self.count_redis = redis
 
         self._changed_packages = set()
 
     def download_counts(self, name):
         # Download Counts from redis
         download_counts = {}
-        if self.redis is not None:
+        if self.count_redis is not None:
             # Get the current utc time
             current = datetime.datetime.utcnow()
 
                 for x in xrange(25)
             ]
             last_1 = sum(
-                [int(x) for x in self.redis.mget(*keys) if x is not None]
+                [int(x) for x in self.count_redis.mget(*keys) if x is not None]
             )
 
             # Get the download count for the last 7 days (roughly)
                 for x in xrange(8)
             ]
             last_7 = sum(
-                [int(x) for x in self.redis.mget(*keys) if x is not None]
+                [int(x) for x in self.count_redis.mget(*keys) if x is not None]
             )
 
             # Get the download count for the last month (roughly)
                 for x in xrange(31)
             ]
             last_30 = sum(
-                [int(x) for x in self.redis.mget(*keys) if x is not None]
+                [int(x) for x in self.count_redis.mget(*keys) if x is not None]
             )
 
             download_counts = {

File tools/rsyslog-cdn.py

 import sys
 import redis
 import csv
+import os
 import posixpath
 import datetime
 import logging
 
 from email.utils import parsedate
 
+# Make sure our PyPI directory is on the sys.path
+root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path = [root] + sys.path
+
+import config
+
+conf = config.Config(os.environ.get("PYPI_CONFIG", os.path.join(root, "config.ini")))
 
 PRECISIONS = [
     ("hour", "%y-%m-%d-%H", datetime.timedelta(days=2)),
 logger.setLevel(logging.DEBUG)
 logger.addHandler(logging.handlers.SysLogHandler(address="/dev/log"))
 
-store = redis.Redis()
+store = redis.Redis.from_url(conf.count_redis_url)
 
 
 def make_key(precision, when, key):

File tools/worker.py

 import config
 
 conf = config.Config(os.environ.get("PYPI_CONFIG", os.path.join(root, "config.ini")))
-redis_conn = redis.Redis.from_url(conf.redis_url)
+redis_conn = redis.Redis.from_url(conf.queue_redis_url)
 
 # Create our queues
 if sys.argv[1:]:
         self.sentry_client = None
         if self.config.sentry_dsn:
             self.sentry_client = raven.Client(self.config.sentry_dsn)
-        if self.config.redis_url:
-            self.redis = redis.Redis.from_url(self.config.redis_url)
-
-            # Queue to handle asynchronous tasks
-            self.queue = rq.Queue(connection=self.redis)
+        if self.config.count_redis_url:
+            self.count_redis = redis.Redis.from_url(self.config.count_redis_url)
         else:
-            self.redis = None
+            self.queue_redis = None
+        if self.config.queue_redis_url:
+            self.queue_redis = redis.Redis.from_url(self.config.queue_redis_url)
+            self.queue = rq.Queue(connection=self.queue_redis)
+        else:
             self.queue = None
         self.env = env
         self.nav_current = None
         self.store = store.Store(
             self.config,
             queue=self.queue,
-            redis=self.redis,
+            redis=self.count_redis,
         )
         try:
             try: