Commits

Evgeniy Tatarkin  committed 6bdfe2e

UrllibAdapterMiddleware manualy put

  • Participants
  • Parent commits e3b4bb8

Comments (0)

Files changed (2)

File pomp/contrib/urllibtools.py

 
     def __init__(self, timeout=5, middlewares=None):
         super(UrllibDownloader, self).__init__(middlewares=middlewares)
-        # insert urllib adpter middleware by default
-        self.middlewares.insert(0, UrllibAdapterMiddleware())
         self.timeout = timeout
 
     def get(self, requests):

File tests/test_contrib_urllib.py

 from pomp.core.base import BaseCrawler, BaseDownloaderMiddleware
 from pomp.core.engine import Pomp
 from pomp.contrib.urllibtools import UrllibDownloader, ThreadedDownloader
+from pomp.contrib.urllibtools import UrllibAdapterMiddleware
 
 from mockserver import HttpServer, make_sitemap
 from tools import DummyCrawler
         collect_middleware = CollectRequestResponseMiddleware()
 
         downloader = ThreadedDownloader(
-            middlewares=[collect_middleware]
+            middlewares=[UrllibAdapterMiddleware(), collect_middleware]
         )
 
         downloader.middlewares.insert(0, req_resp_midlleware)
         catch_exception_middleware = CatchException()
         pomp = Pomp(
             downloader=UrllibDownloader(
-                middlewares=[catch_exception_middleware]),
+                middlewares=[
+                    UrllibAdapterMiddleware(),
+                    catch_exception_middleware
+                ]),
             pipelines=[],
         )