Commits

Xiangjun Zhou committed 0e3a2c4 Draft

update

Comments (0)

Files changed (1)

 
 logging.basicConfig(level=logging.DEBUG)
 
+'''
 urls = ['http://www.163.com', 'http://www.qq.com', 'http://www.sina.com.cn',
         'http://www.sohu.com', 'http://www.yahoo.com', 'http://www.baidu.com',
         'http://www.apple.com', 'http://www.microsoft.com']
+urls = ['http://www.nytimes.com']
+'''
 
+urls = ['http://news.sina.com.cn/society/']
 
 class Crawler:
     def parser(self, req_url, data):
+        print data
         return [len(data)]
 
     def pipeline(self, response):
     def testCrawler(self):
         dt = datetime.now()
         crawler = Crawler()
-        Scheduler(urls, crawler.parser, crawler.pipeline, 8)
+        Scheduler(urls, crawler.parser, crawler.pipeline, max_running=8)
         print datetime.now() - dt
 
-
 unittest.main()