Evgeniy Tatarkin avatar Evgeniy Tatarkin committed d922278

chain urls for _width_first_method

Comments (0)

Files changed (1)

pomp/core/engine.py

 """
 Engine
 """
+import itertools
 from pomp.core.utils import iterator
 
 
         else:
             return urls
 
-
     def pump(self, crawler):
 
         map(lambda pipe: pipe.start(), self.pipelines)
 
         try:
-            next_urls = list(self.downloader.process(
+            next_urls = self.downloader.process(
                 iterator(crawler.ENTRY_URL),
                 self.response_callback,
                 crawler
-            ))
+            )
 
             if not crawler.is_depth_first:
                 for urls in next_urls:
-                    next_urls += list(self.downloader.process(
-                        iterator(urls),
-                        self.response_callback,
-                        crawler
-                    ))
+                    next_urls = itertools.chain(
+                        next_urls,
+                        self.downloader.process(
+                            iterator(urls),
+                            self.response_callback,
+                            crawler
+                        )
+                    )
 
         finally:
             map(lambda pipe: pipe.stop(), self.pipelines)
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.