Sebastian Rahlf avatar Sebastian Rahlf committed f5888d6

Added RetryAPI to contrib package.

Comments (0)

Files changed (3)

amazonproduct/api.py

 from time import strftime, gmtime, sleep
 import urllib2
 import warnings
-import math
-import time
-import re
 
 # For historic reasons, this module also supports Python 2.4. To make this
 # happen, a few things have to be imported differently, e.g. pycrypto is needed
 from amazonproduct.utils import load_config, running_on_gae, REQUIRED_KEYS
 from amazonproduct.processors import ITEMS_PAGINATOR
 
+
 # load default processor
 try:
     from amazonproduct.processors.objectify import Processor
     'us': ('ecs.amazonaws.com', 'xml-us.amznxslt.com'),
 }
 
-def retry(tries=5, delay=3, backoff=1):
-    """Retry decorator
-    adapted from http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
-    Kwargs:
-        tries (int): max number of tries before giving up
-        delay (int): delay between tries in seconds
-        backoff (int): exponential backoff multiplier
-    """
-    tries = math.floor(tries)
-    if tries < 0:
-        raise ValueError("tries must be 0 or greater")
-    if delay <= 0:
-        raise ValueError("delay must be greater than 0")
-    if backoff < 1:
-        raise ValueError("backoff must be 1 or greater")
-    def deco_retry(f):
-        def f_retry(*args, **kwargs):
-            mtries, mdelay = tries, delay # make mutable
-            while mtries > 0:
-                try:
-                    return f(*args, **kwargs)
-                except (urllib2.URLError, socket.timeout), e:
-                    # following "urllib2 - The Missing Manual" example
-                    # http://www.voidspace.org.uk/python/articles/urllib2.shtml#id22
-                    if hasattr(e, 'reason'):
-                        error_msg = str(e.reason)
-                    elif hasattr(e, 'message'):
-                        error_msg = str(e.message)
-                    else:
-                        raise
-                    if not re.search(r'timed out', error_msg, re.I):
-                        raise
-                    if mtries == 1:
-                        msg = "%s, Retry maxed, Gave up" % str(e)
-                    else:
-                        msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
-                    print(msg)
-                    if mtries == 1:
-                        break
-                    time.sleep(mdelay)
-                    mdelay *= backoff
-                    mtries -= 1
-                except:
-                    raise
-            raise
-        return f_retry # true decorator
-    return deco_retry
-
 
 class GZipHandler(urllib2.BaseHandler):
 
             # otherwise simply re-raise
             raise
 
-    @retry()
     def call(self, **qargs):
         """
         Builds a signed URL for the operation, fetches the result from Amazon

amazonproduct/contrib/retry.py

+
+import socket
+import time
+import urllib2
+
+from amazonproduct.api import API
+
+class RetryAPI (API):
+
+    """
+    API which will try up to TRIES times to fetch a result from Amazon should it run into a timeout.
+
+    Based on work by Jerry Ji
+    """
+
+    #: Max number of tries before giving up
+    TRIES = 5
+
+    #: Delay between tries in seconds
+    DELAY = 3
+
+    #: Between each try the delay will be lengthened by this backoff multiplier
+    BACKOFF = 1
+
+    def _fetch(self, url):
+        """
+        Retrieves XML response from Amazon. In case of a timeout, it will try
+        :const:`~RetryAPI.TRIES`` times before raising an error.
+        """
+        attempts = 0
+        delay = self.DELAY
+
+        while True:
+            try:
+                attempts += 1
+                return API._fetch(self, url)
+            except urllib2.URLError, e:
+
+                # if a timeout occurred
+                # wait for some time before trying again
+                reason = getattr(e, 'reason', None)
+                if isinstance(reason, socket.timeout) and attempts < self.TRIES:
+                    time.sleep(delay)
+                    delay *= self.BACKOFF
+                    continue
+
+                # otherwise reraise the original error
+                raise
+

tests/test_contrib_retry.py

+import socket
+from urllib2 import URLError
+import pytest
+
+from amazonproduct.api import API
+from amazonproduct.contrib.retry import RetryAPI
+
+@pytest.mark.slowtest
+def test_timeout(monkeypatch):
+    """
+    Check that in case of a timeout API will not give up easily.
+    """
+
+    class mock_fetch (object):
+        def __init__(self):
+            self.calls = 0
+        def __call__(self, _, url):
+            self.calls += 1
+            print 'call %i: %s' % (self.calls, url)
+            raise URLError(socket.timeout())
+
+    fetcher = mock_fetch()
+    monkeypatch.setattr(API, '_fetch', fetcher)
+
+    api = RetryAPI(locale='de')
+
+    itworked = False
+    try:
+        api.call(operation='DummyOperation')
+        itworked = True
+    except URLError, e:
+        assert isinstance(e.reason, socket.timeout)
+
+    # timeout WAS raised and fetch was called TRIES times
+    assert not itworked
+    assert fetcher.calls == api.TRIES
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.