Commits

Evgeniy Tatarkin committed 554be87

py3k support

  • Participants
  • Parent commits dc6909c

Comments (0)

Files changed (5)

examples/pythonnews.py

     ENTRY_URL = 'http://python.org/news/'
 
     def extract_items(self, url, page):
-
+        page = page.decode('utf-8')
         for i in news_re.findall(page):
             item = PythonNewsItem()
             item.title, item.published = i[0], i[2]

pomp/contrib/__init__.py

 """
 Standart downloaders
 """
-import urllib2
+try:
+    from urllib.request import urlopen
+except ImportError:
+    from urllib2 import urlopen
+
 from pomp.core.base import BaseDownloader
 
 
     TIMEOUT = 5
 
     def get(self, url):
-        response = urllib2.urlopen(url, timeout=self.TIMEOUT)
+        response = urlopen(url, timeout=self.TIMEOUT)
         return response.read()

pomp/core/base.py

 class BaseDownloader(object):
 
     def process(self, urls, callback, crawler):
+        # return next urls
         return filter(
             None,
-            map(lambda url: callback(crawler, url, self.get(url)), urls)
+            list(map(lambda url: callback(crawler, url, self.get(url)), urls))
         )
 
     def get(self, url):

pomp/core/engine.py

             for pipe in self.pipelines:
                 items = filter(None, map(pipe.process, items))
 
+                # launch items pipelinse process
+                items = list(items)
+
         urls = crawler.next_url(page)
         if crawler.is_depth_first:
             if urls:
 
     def pump(self, crawler):
 
-        map(lambda pipe: pipe.start(), self.pipelines)
+        for pipe in self.pipelines:
+            pipe.start()
 
         try:
             next_urls = self.downloader.process(
                     )
 
         finally:
-            map(lambda pipe: pipe.stop(), self.pipelines)
+
+            for pipe in self.pipelines:
+                pipe.stop() 

pomp/core/utils.py

     if isinstance(var, types.GeneratorType):
         return var
 
-    if isinstance(var, types.ListType) or isinstance(var, types.TupleType):
+    if isinstance(var, list) or isinstance(var, tuple):
         return iter(var)
 
     return iter((var,))