Commits

Gregory Petukhov  committed 93856b8

Refactor Data processing: now it is possible to pass multiple object in one Data instance

  • Participants
  • Parent commits c9e748b

Comments (0)

Files changed (3)

File grab/spider/base.py

         elif isinstance(result, Data):
             handler = self.find_data_handler(result)
             try:
-                handler(result.item)
+                handler(**result.storage)
             except Exception, ex:
                 self.process_handler_error('data_%s' % result.name, ex, task)
         elif result is None:

File grab/spider/data.py

     Task handlers should return instances of that class.
     """
 
-    def __init__(self, name, item):
-        self.name = name
-        self.item = item
+    def __init__(self, data_name, **kwargs):
+        self.name = data_name
+        self.storage = kwargs

File test/spider_data.py

     def test_data_nohandler_error(self):
         class TestSpider(Spider):
             def task_page(self, grab, task):
-                yield Data('foo', 1)
+                yield Data('foo', num=1)
 
         bot = TestSpider()
         bot.setup_queue()
     def test_exception_from_data_handler(self):
         class TestSpider(Spider):
             def task_page(self, grab, task):
-                yield Data('foo', 1)
+                yield Data('foo', num=1)
             
             def data_foo(self, num):
                 1/0
         bot.add_task(Task('page', url=SERVER.BASE_URL))
         bot.run()
         self.assertTrue('data_foo' in bot.items['fatal'][0])
+
+    def test_data_simple_case(self):
+        class TestSpider(Spider):
+            def prepare(self):
+                self.data_processed = []
+
+            def task_page(self, grab, task):
+                yield Data('foo', number=1)
+            
+            def data_foo(self, number):
+                self.data_processed.append(number)
+
+        bot = TestSpider()
+        bot.setup_queue()
+        bot.add_task(Task('page', url=SERVER.BASE_URL))
+        bot.run()
+        self.assertEqual(bot.data_processed, [1])
+
+    def test_complex_data(self):
+        class TestSpider(Spider):
+            def prepare(self):
+                self.data_processed = []
+
+            def task_page(self, grab, task):
+                yield Data('foo', one=1, two=2, bar='gaz')
+            
+            def data_foo(self, one, two, **kwargs):
+                self.data_processed.append(one)
+                self.data_processed.append(two)
+                self.data_processed.append(kwargs)
+
+        bot = TestSpider()
+        bot.setup_queue()
+        bot.add_task(Task('page', url=SERVER.BASE_URL))
+        bot.run()
+        self.assertEqual(bot.data_processed, [1, 2, {'bar': 'gaz'}])