Commits

Evgeniy Tatarkin committed 83c30ab

add simple threaded downloader

Comments (0)

Files changed (1)

pomp/contrib/__init__.py

 except ImportError:
     from urllib2 import urlopen
 
+from multiprocessing.pool import ThreadPool
 from pomp.core.base import BaseDownloader
 
 
     def get(self, url):
         response = urlopen(url, timeout=self.TIMEOUT)
         return response.read()
+
+
+class ThreadedDownloader(SimpleDownloader):
+
+    def __init__(self, pool_size=5):
+        self.workers_pool = ThreadPool(processes=pool_size)
+
+    def process(self, urls, callback, crawler):
+        pages = self.workers_pool.map(self.get, urls)
+        return filter(
+            None,
+            list(map(lambda res: callback(crawler, None, res), pages))
+        )