Commits

Drew Smathers  committed 41e2c49

cooperatively iterate on the main chunking process

  • Participants
  • Parent commits 4e1869f

Comments (0)

Files changed (1)

     from twisted.internet import reactor
     from twisted.internet.defer import gatherResults
     from twisted.internet.threads import deferToThread
+    from twisted.internet.task import coiterate
     thread_pool = 1
 except ImportError:
     thread_pool = 0
         reactor.stop()
     gatherResults(deferreds).addErrback(log.err).addCallback(end)
 
-    for (piece, fd) in enumerate(generate_chunk_files(path), 1):
-        q.put((fd, piece))
-    for i in range(threads):
-        q.put(DONE)
+    def chunker():
+        for (piece, fd) in enumerate(generate_chunk_files(path), 1):
+            q.put((fd, piece))
+            yield
+        for i in range(threads):
+            q.put(DONE)
+            yield
+    chunker = chunker()
+    return coiterate(chunker).addCallback(
+        lambda ign: sys.stderr.write('\nfinished generating chunks for threads\n'))
 
 def main():
     parser = optparse.OptionParser()