Tarek Ziadé avatar Tarek Ziadé committed c848dea Draft

fixed a few things

Comments (0)

Files changed (2)

pep381client/__init__.py

 import xml.parsers.expat
 import sqlite
 import threading
+import Queue
+import thread
+import signal
 from httplib import (IncompleteRead, BadStatusLine, CannotSendRequest,
                      ResponseNotReady)
 import socket
         _proxy.useragent = UA
     return _proxy
 
-_conn = None
+_conn = {}
+
 def http():
-    global _conn
-    if _conn is None:
-        _conn = httplib.HTTPConnection(pypi)
-        _conn.connect()
+    ident = thread.get_ident()
+
+    if ident not in _conn:
+        conn = httplib.HTTPConnection(pypi)
+        conn.connect()
+        _conn[ident] = conn
+    else:
+        conn = _conn[ident]
+
     # check that connection is still open
     try:
-        if not _conn.sock:
+        if not conn.sock:
             # HTTP server had announced to close the connection
             raise socket.error
-        _conn.sock.getpeername()
+        conn.sock.getpeername()
     except socket.error:
-        _conn = httplib.HTTPConnection(pypi)
-        _conn.connect()
-    return _conn
+        conn = httplib.HTTPConnection(pypi)
+        conn.connect()
+    return conn
 
 def now():
     return int(time.time())
     def __init__(self, sync, queue):
         self.sync = sync
         self.queue = queue
+        self.running = False
         threading.Thread.__init__(self)
+        self.daemon = True
 
     def run(self):
-        while not queue.empty():
-            project = queue.get()
+        self.running = True
+        while not self.queue.empty() and self.running:
             try:
-                self.sync._synchronize(project)
+                project = self.queue.get(timeout=5)
+            except Queue.Empty:
+                return
+            try:
+                self.sync._synchronize_project(project)
             except Exception, e:
-                print "Failed on %s" % project
+                print "Failed on %s" % project.encode('utf-8')
                 print str(e)
 
 
+lock = threading.RLock()
+
+
 class Synchronization:
     "Picklable status of a mirror"
 
         self.skip_file_contents = False
 
         self.filter_external_hrefs = True
-        self.lock = threading.RLock()
 
     def defaults(self):
         # Fill fields that may not exist in the pickle
                 setattr(self, field, value)
 
     def store(self):
-        with self.lock:
+        with lock:
             with open(os.path.join(self.homedir, "status"), "wb") as f:
                 cPickle.dump(self, f, cPickle.HIGHEST_PROTOCOL)
                 self.storage.commit()
     @staticmethod
     def load(homedir, storage=None):
         if not os.path.exists(os.path.join(homedir, "status")):
+            build = True
+        try:
+            res = cPickle.load(open(os.path.join(homedir, "status"), "rb"))
+            build = False
+        except EOFError:
+            print "File exists but seem broken, rebuilding it"
+            build = True
+
+        if build:
+            print "Building a status - this may be long..."
             status = Synchronization()
             status.homedir = homedir
             status.last_started = now()
             status.store()
             return status
 
-
-        res = cPickle.load(open(os.path.join(homedir, "status"), "rb"))
         res.storage = storage or sqlite.SqliteStorage(os.path.join(homedir, "files"))
         res.defaults()
         res.homedir = homedir # override pickled value in case it got relocated
         for d in ('simple', 'packages', 'serversig',
                   'local-stats/days'):
             os.makedirs(os.path.join(targetdir, 'web', d))
+        if not self.quiet:
+            print "Building a status - this may be long..."
         status = Synchronization()
         status.homedir = targetdir
         status.last_started = now()
             worker.start()
 
         # waiting for them to end the job
-        for worker in workers:
-            worker.join()
+        is_alive = 10
+
+        while is_alive > 0:
+            count = 0
+            for worker in workers:
+                if worker.isAlive():
+                    count += 1
+            is_alive = count
+            time.sleep(2)
 
         self.update_timestamp(self.last_started)
         self.last_completed = self.last_started
         self.last_started = 0
         self.store()
 
-    def _synchronize(project):
+    def _synchronize_project(self, project):
 
         if not self.quiet:
             print "Synchronizing", project.encode('utf-8')
 
         try:
             data = self.copy_simple_page(project)
-        except ValueError:
+        except ValueError, e:
             if not self.quiet:
                 print ("Could not copy simple page for %s -- will retry"
                         % project)
+                print str(e)
             return
 
         if not data:
                  h.putrequest('GET', '/simple/'+urllib2.quote(project)+'/')
             else:
                  h.putrequest('GET', '/simple/')
-        except CannotSendRequest:
-            raise ValueError, "Cannot reach PyPI"
+        except CannotSendRequest, e:
+            raise ValueError, "Cannot reach PyPI " + str(e)
 
         h.putheader('User-Agent', UA)
         h.endheaders()

pep381client/sqlite.py

 import sqlite3, os
+import threading
+import thread
+
+lock = threading.RLock()
+connections = {}
+
 
 class SqliteStorage(object):
 
               ]
 
     def __init__(self, filename):
-        self.conn = sqlite3.connect(filename)
+        self.conns = {}
+        self.filename = filename
         cursor = self.conn.cursor()
         for stmt in self.schema:
             cursor.execute(stmt)
         self.commit()
 
+    @property
+    def conn(self):
+        ident = thread.get_ident()
+        if ident in self.conns:
+            return self.conns[ident]
+        else:
+            return sqlite3.connect(self.filename)
+
     def commit(self):
         self.conn.commit()
 
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.