Commits

Evgeniy Tatarkin  committed c48ae41

utils.DeferredList works with deferreds and non-deferreds objects

  • Participants
  • Parent commits 232b585

Comments (0)

Files changed (4)

 
 Here you can see the full list of changes between pomp releases.
 
+
+Version 0.1.2
+-------------
+
+Not released yet
+
+- utils.DeferredList works with deferred and not-deferred objects
+
+
 Version 0.1.1
 -------------
 
-Not released yet
+Released on December 12nd 2013
 
 - urllib and twidted downloader on get method yield result
 - bugfix `depth first` method

File pomp/__init__.py

-__version__ = (0, 1, 1, 'dev', 0)
+__version__ = (0, 1, 2, 'dev', 0)

File pomp/core/engine.py

         return self.stop_deferred
 
     def _sync_or_async(self, next_requests, crawler, callback):
-        # separate deferred and regular requests
-        # fire generator
-        deferreds = []
-        other = []
-        for r in filter_requests(next_requests):
-            if isinstance(r, defer.Deferred):
-                deferreds.append(r)
-            else:
-                other.append(r)
-
-        if deferreds:  # async behavior
-            d = DeferredList(deferreds)
-            d.add_callback(callback, crawler)
-            return d
-
-        if other:  # sync behavior
-            return callback(other, crawler)
+        d = DeferredList([r for r in filter_requests(next_requests)])
+        d.add_callback(callback, crawler)
+        return d
 
     def _do(self, requests, crawler):
         # execute request by downloader

File pomp/core/utils.py

         self.finished_count = 0
 
         for index, deferred in enumerate(deferredList):
-            deferred.add_callbacks(
-                self._cb_deferred,
-                self._cb_deferred,
-                callback_args=(index,),
-                errback_args=(index,)
-            )
+            if isinstance(deferred, defer.Deferred):
+                deferred.add_callbacks(
+                    self._cb_deferred,
+                    self._cb_deferred,
+                    callback_args=(index,),
+                    errback_args=(index,)
+                )
+            else:  # if request allready done
+                self.finished_count += 1
+                self.result_list[index] = deferred
+
+        # check are deferreds or results was passed to __init__
+        try:
+            self.check_and_fire()
+        except defer.AlreadyCalledDeferred:
+            pass
 
     def _cb_deferred(self, result, index):
         """(internal) Callback for when one of my deferreds fires.
         self.result_list[index] = result
 
         self.finished_count += 1
+        self.check_and_fire()  # check is done and fire
+        return result
+
+    def check_and_fire(self):
         if self.finished_count == len(self.result_list):
             self.callback(self.result_list)
-
-        return result