Commits

brianfrantz committed fc93d38

Retry memcached lock release up to 5 times. Give up waiting on memcached lock acquire after 5 attempts. Otherwise a release failure will cause a permanent lockup on future lock acquires.

Comments (0)

Files changed (1)

dogpile/cache/backends/memcached.py

 
     def acquire(self, wait=True):
         client = self.client_fn()
-        i = 0
-        while True:
+        for i in xrange(5):  # Try 5 times to acquire the lock
             if client.add(self.key, 1):
                 return True
             elif not wait:
                 return False
             else:
-                sleep_time = (((i+1)*random.random()) + 2**i) / 2.5
+                sleep_time = 1 + (2*random.random())  # Sleep for 1-3 seconds
+                print "------------ CACHE LOCK SLEEP %s" % sleep_time
                 time.sleep(sleep_time)
-            if i < 15:
-                i += 1
+        print "------------ CACHE LOCK GIVING UP ON WAITING"
+        return False
 
     def release(self):
         client = self.client_fn()
-        client.delete(self.key)
+        for i in xrange(5):  # Try 5 times to release the lock
+            if client.delete(self.key):
+                return
+            sleep_time = 2*random.random()  # Sleep for 0-2 seconds
+            print "------------ CACHE LOCK DELETE %s FAILED.  SLEEP %s" % (i, sleep_time)
+            time.sleep(sleep_time)
+        print "------------ CACHE LOCK DELETE FAILED EVEN AFTER RETRIES"
 
 class GenericMemcachedBackend(CacheBackend):
     """Base class for memcached backends.