Commits

Gregory Petukhov committed 0f58174

Fix the bug: fallback method is not called when tasks network try limit is reached

Comments (0)

Files changed (2)

grab/spider/base.py

 
         return is_valid
 
+    def process_task_fallback(self, task):
+        try:
+            fallback_handler = getattr(self, 'task_%s_fallback' % task.name)
+        except AttributeError:
+            pass
+        else:
+            logger.error('task_*_fallback methods are deprecated! Do not use this feature please. It will be replaced with middleware layer')
+            fallback_handler(task)
+
     def check_task_limits_deprecated(self, task):
         is_valid = self.check_task_limits(task)
 
         if not is_valid:
-            try:
-                fallback_handler = getattr(self, 'task_%s_fallback' % task.name)
-            except AttributeError:
-                pass
-            else:
-                logger.error('task_*_fallback methods are deprecated! Do not use this feature please. It will be replaced with middleware layer')
-                fallback_handler(task)
+            self.process_task_fallback(task)
 
         return is_valid
 
 
                         if not self.check_task_limits(task):
                             logger_verbose.debug('Task %s is rejected due to limits' % task.name)
+                            self.process_task_fallback(task)
                         else:
                             self.process_new_task(task)
 

test/spider_task.py

 
         #bot.run()
         #self.assertEqual(['func'], tokens)
+
+    def test_task_fallback(self):
+        class TestSpider(Spider):
+            def prepare(self):
+                self.tokens = []
+
+            def task_page(self, grab, task):
+                self.tokens.append('task')
+
+            def task_page_fallback(self, task):
+                self.tokens.append('fallback')
+
+        SERVER.RESPONSE['code'] = 403
+        bot = TestSpider(network_try_limit=2)
+        bot.setup_queue()
+        bot.add_task(Task('page', url=SERVER.BASE_URL))
+        bot.run()
+        self.assertEqual(bot.tokens, ['fallback'])