Evgeniy Tatarkin avatar Evgeniy Tatarkin committed 6bdfe2e

UrllibAdapterMiddleware manualy put

Comments (0)

Files changed (2)


     def __init__(self, timeout=5, middlewares=None):
         super(UrllibDownloader, self).__init__(middlewares=middlewares)
-        # insert urllib adpter middleware by default
-        self.middlewares.insert(0, UrllibAdapterMiddleware())
         self.timeout = timeout
     def get(self, requests):


 from pomp.core.base import BaseCrawler, BaseDownloaderMiddleware
 from pomp.core.engine import Pomp
 from pomp.contrib.urllibtools import UrllibDownloader, ThreadedDownloader
+from pomp.contrib.urllibtools import UrllibAdapterMiddleware
 from mockserver import HttpServer, make_sitemap
 from tools import DummyCrawler
         collect_middleware = CollectRequestResponseMiddleware()
         downloader = ThreadedDownloader(
-            middlewares=[collect_middleware]
+            middlewares=[UrllibAdapterMiddleware(), collect_middleware]
         downloader.middlewares.insert(0, req_resp_midlleware)
         catch_exception_middleware = CatchException()
         pomp = Pomp(
-                middlewares=[catch_exception_middleware]),
+                middlewares=[
+                    UrllibAdapterMiddleware(),
+                    catch_exception_middleware
+                ]),
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.