Commits

Evgeniy Tatarkin  committed 86f5354

remove collector from engine

  • Participants
  • Parent commits 190d229

Comments (0)

Files changed (2)

File pomp/core/engine.py

 
         self.downloader = downloader
         self.pipelines = pipelines or tuple()
-        self.collector = []
-
-    def collect(self, items):
-        self.collector += items
 
     def response_callback(self, crawler, url, page):
 
             for pipe in self.pipelines:
                 items = filter(None, map(pipe.process, items))
 
-        self.collect(items)
-
         urls = crawler.next_url(page)
 
         if urls:
                 self.response_callback,
                 crawler
             )
-
-            return self.collector
         finally:
             map(lambda pipe: pipe.stop(), self.pipelines)

File tests/test_simple_crawler.py

                     return None
                 return item
 
+        class SavePipeline(BasePipeline):
+
+            def __init__(self, collection):
+                self.collection = collection
+
+            def process(self, item):
+                self.collection.append(item)
+                return item
+
+        result = []
+
         pomp = Pomp(
             downloader=DummyDownloader(),
-            pipelines=[IncPipeline(), FilterPipeline()],
+            pipelines=[
+                IncPipeline(),
+                FilterPipeline(),
+                SavePipeline(result),
+            ],
         )
 
-        result = pomp.pump(DummyCrawler())
+        pomp.pump(DummyCrawler())
 
         assert len(result) == 2