Commits

Tarek Ziadé  committed 955cb5e

added some except catches

  • Participants
  • Parent commits 9d55326

Comments (0)

Files changed (1)

File pep381client/__init__.py

 from xml.etree import ElementTree
 import xml.parsers.expat
 import sqlite
+from httplib import (IncompleteRead, BadStatusLine, CannotSendRequest,
+                     ResponseNotReady)
+import socket
 
 # library config
 pypi = 'pypi.python.org'
     def __init__(self):
         self.homedir = None
         self.quiet = False
-        
+
         # time stamps: seconds since 1970
         self.last_completed = 0 # when did the last run complete
         self.last_started = 0   # when did the current run start
 
     @staticmethod
     def load(homedir, storage=None):
+        if not os.path.exists(os.path.join(homedir, "status")):
+            status = Synchronization()
+            status.homedir = homedir
+            status.last_started = now()
+            status.projects_to_do = set(xmlrpc().list_packages())
+            status.storage = storage or sqlite.SqliteStorage(os.path.join(status.homedir, "files"))
+            status.store()
+            return status
+
+
         res = cPickle.load(open(os.path.join(homedir, "status"), "rb"))
         res.storage = storage or sqlite.SqliteStorage(os.path.join(homedir, "files"))
         res.defaults()
         'Run synchronization. Can be interrupted and restarted at any time.'
         if self.last_started == 0:
             # no synchronization in progress. Fetch changelog
+            if not self.quiet:
+                print "Fetching the changelog"
             self.last_started = now()
             changes = xmlrpc().changelog(self.last_completed-1)
             if not changes:
                 self.projects_to_do.add(change[0])
             self.copy_simple_page('')
             self.store()
+
+        if not self.quiet:
+            print "Starting the mirror sync."
+
         # sort projects to allow for repeatable runs
         for project in sorted(self.projects_to_do):
             if not project:
                 continue
             if not self.quiet:
                 print "Synchronizing", project.encode('utf-8')
-            data = self.copy_simple_page(project)
+
+            try:
+                data = self.copy_simple_page(project)
+            except ValueError:
+                if not self.quiet:
+                    print ("Could not copy simple page for %s -- will retry"
+                           % project)
+                continue
+
             if not data:
                 self.delete_project(project)
                 self.store()
                 continue
+
             try:
                 files = set(self.get_package_files(data))
             except xml.parsers.expat.ExpatError, e:
                 if not self.quiet:
                     print "Page for %s cannot be parsed: %r" % (project, e)
                 raise
+
+            copy_failed = False
             for file in files:
                 if not self.quiet:
                     print "Copying", file
-                self.maybe_copy_file(project, file)
+                try:
+                    self.maybe_copy_file(project, file)
+                except IOError:
+                    # we need to try again
+                    copy_failed = True
+
+            if copy_failed:
+                continue
+
             # files start with /; remove it
             relfiles = set(p[1:] for p in files)
             for file in self.storage.files(project)-relfiles:
     def copy_simple_page(self, project):
         project = project.encode('utf-8')
         h = http()
-        if project:
-             h.putrequest('GET', '/simple/'+urllib2.quote(project)+'/')
-        else:
-             h.putrequest('GET', '/simple/')
+        try:
+            if project:
+                 h.putrequest('GET', '/simple/'+urllib2.quote(project)+'/')
+            else:
+                 h.putrequest('GET', '/simple/')
+        except CannotSendRequest:
+            raise ValueError, "Cannot reach PyPI"
+
         h.putheader('User-Agent', UA)
         h.endheaders()
-        r = h.getresponse()
+
+        try:
+            r = h.getresponse()
+        except BadStatusLine:
+            raise ValueError, "Cannot read PyPI response"
+
         html = r.read()
         if r.status == 404:
             return None
         h.putrequest('GET', '/serversig/'+urllib2.quote(project)+'/')
         h.putheader('User-Agent', UA)
         h.endheaders()
-        r = h.getresponse()
+        try:
+            r = h.getresponse()
+        except BadStatusLine:
+            raise ValueError, "Cannot read PyPI response"
+
         sig = r.read()
         if r.status != 200:
             if not project:
 
     def maybe_copy_file(self, project, path):
         h = http()
-        if self.skip_file_contents:
-            h.putrequest("HEAD", urllib2.quote(path))
-        else:
-            h.putrequest("GET", urllib2.quote(path))
+        try:
+            if self.skip_file_contents:
+                h.putrequest("HEAD", urllib2.quote(path))
+            else:
+                h.putrequest("GET", urllib2.quote(path))
+        except CannotSendRequest:
+            raise IOError, "Could not send a request -- will retry in the next pass"
+
         h.putheader('User-Agent', UA)
         etag = self.storage.etag(path)
         if etag:
             h.putheader("If-none-match", etag)
         h.endheaders()
-        r = h.getresponse()
+        try:
+            r = h.getresponse()
+        except (BadStatusLine, socket.error, ResponseNotReady):
+            raise IOError, "Received a bad status -- will retry in the next pass"
+
         if r.status == 304:
             # not modified, discard data
             r.read()
         lpath = os.path.join(self.homedir, "web", path)
         if r.status == 200:
             self.storage.remove_file(path) # readd when done downloading
-            data = r.read()
+            try:
+                data = r.read()
+            except IncompleteRead:
+                # was unable to read the whole file
+                if not self.quiet:
+                    print ("Was unable to get the whole file -- "
+                           "will retry in the next pass")
+                self.remove_file(path)
+                return
+
             dirname = os.path.dirname(lpath)
             if not os.path.exists(dirname):
                 os.makedirs(dirname)