Source

distribute / warn-on-http-error

Full commit
# HG changeset patch
# Parent 3bf82d0cbc6e0a6d383f177c4def1222f3c3ba46

diff -r 3bf82d0cbc6e setuptools/package_index.py
--- a/setuptools/package_index.py	Sun May 19 12:58:37 2013 -0400
+++ b/setuptools/package_index.py	Thu Jun 06 20:39:27 2013 +0100
@@ -203,6 +203,13 @@
         if f is None: return
         self.fetched_urls[url] = self.fetched_urls[f.url] = True
 
+        if isinstance(f, urllib2.HTTPError):
+            if f.code != 404:
+                self.warn("Download error on %s: %s %s", url, f.code, f.msg)
+                self.process_index(url, "")
+            f.close()   # don't try to process HTTP error pages
+            return
+
         if 'html' not in f.headers.get('content-type', '').lower():
             f.close()   # not html, we can't process it
             return
@@ -210,17 +217,13 @@
         base = f.url     # handle redirects
         page = f.read()
         if not isinstance(page, str): # We are in Python 3 and got bytes. We want str.
-            if isinstance(f, urllib2.HTTPError):
-                # Errors have no charset, assume latin1:
-                charset = 'latin-1'
-            else:
-                charset = f.headers.get_param('charset') or 'latin-1'
+            charset = f.headers.get_param('charset') or 'latin-1'
             page = page.decode(charset, "ignore")
         f.close()
         for match in HREF.finditer(page):
             link = urlparse.urljoin(base, htmldecode(match.group(1)))
             self.process_url(link)
-        if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
+        if url.startswith(self.index_url):
             page = self.process_index(url, page)
 
     def process_filename(self, fn, nested=False):