Commits

Anonymous committed 575de02

Better REDIS-enabled caching strategy

  • Participants
  • Parent commits 1d30962

Comments (0)

Files changed (1)

storages/backends/moka.py

-from queued_storage.backend import QueuedRemoteStorage
+import os
 import logging
 
 from django.conf import settings
-
+from queued_storage.backend import QueuedRemoteStorage
 from redis import Redis  # depends on Redis server
 
 # Get an instance of a logger
 logger.setLevel(logging.INFO)
 
 REDIS = getattr(settings, 'REDIS', None)
-REMOTE = "remote"  # local const
+
+
+def upload_to(self, filename):
+    """(Missing docstring)"""
+    # workaround - cloudfiles backend currently suppresses '/'
+    fullpath = unicode(os.path.join(self.path, filename)).\
+        replace("/", "-")
+
+    return fullpath
+
 
 class MokaStorage(QueuedRemoteStorage):
     """Moka custom storage."""
             remote='storages.backends.mosso.CloudFilesStorage')
 
         self._redis = Redis(**settings.REDIS["storage"])
-        logger.info("Connection to redis is now active: %s", self._redis)
+        logger.info("Connection to redis is now active: %s.", self._redis)
+
+    def _get_local_storage(self, name):
+
+        if (self.local.exists(name)):
+            logger.info(
+                "Retrying upload for file '%s'", name
+            )
+            self._send_task(name)
+
+        else:
+            # ok, here we're in trouble :-(
+            logger.error(
+                "File '%s' is missing from both local and remote resource", name
+            )
+
+        return self.local
 
     def get_storage(self, name):
-        cache_result = self._redis.sismember(REMOTE, name)
+        cache_result = self._redis.sismember(settings.REMOTE, name)
 
         if cache_result:
-            logger.info("Cache hit for file '%s'", name)
+            logger.info("Cache hit ok for file '%s'", name)
             return self.remote
 
-        elif self.remote.exists(name):
-            logger.info("Caching remote storage for file '%s'", name)
-            self._redis.sadd(REMOTE, name)
+        elif self._redis.sismember(settings.MISSING, name):
+            logger.info(
+                "File '%s' known to be missing from the remote resource. ",
+                name
+            )
+            return self._get_local_storage(name)
+
+        elif self.remote.exists(name):  # quite expensive
+            logger.info("Caching remote storage for file '%s'.", name)
+            self._redis.sadd(settings.REMOTE, name)
             return self.remote
 
-        logger.info("File '%s' was not found on the remote resource", name)
-        return self.local
+        else:
+            self._redis.sadd(settings.MISSING, name)
+            logger.warning(
+                "File '%s' was not found on the remote resource", name
+            )
+            return self._get_local_storage(name)