1. Ned Batchelder
  2. lifeboat

Commits

Ned Batchelder  committed 73b6612 Draft

More tweaks

  • Participants
  • Parent commits 04f368f
  • Branches default

Comments (0)

Files changed (1)

File save_tabblos.py

View file
                         continue
 
                     # Didn't finish it, add back into the queue to try again later
-                    todo.put((id, i, num_retries+1))
+                    if num_retries < 20:
+                        todo.put((id, i, num_retries+1))
+                    else:
+                        self.log("Tried %s times, skipping %d." % (num_retries, id))
+                        self.harv.skipped.append(id)
 
         downloaders = [DownloaderThread(tid, self) for tid in range(self.num_threads)]
         for dl in downloaders:
                     break
                 zipdata.write(buf)
                 sofar += len(buf)
-                self.log("Read %d bytes.." % sofar)
+                self.log("Read %dM bytes.." % (sofar//(1024*1024)))
             try:
                 zipf = zipfile.ZipFile(zipdata)
             except Exception, e:
         ids = self.get_ids()
         self.log("Found %d tabblos" % len(ids))
         self.download_tabblos(ids)
-        if self.mine and self.skipped:
+        if self.skipped:
             self.log("Skipped %d tabblos: %s." % (len(self.skipped), ", ".join(str(i) for i in self.skipped)))
-            self.log("Correct those tabblos (if you haven't already deleted them) and rerun Lifeboat.")
-            os.remove(self.ids_txt_fname)
+            if self.mine:
+                self.log("Correct those tabblos (if you haven't already deleted them) and rerun Lifeboat.")
+                os.remove(self.ids_txt_fname)
 
         ids = sorted(set(ids) - set(self.skipped))
         self.generate_toc(ids)