Commits

Ada Young committed 43e59c2

Added some experimental code to LoadData() for faster bulk news import

Comments (0)

Files changed (1)

         orgid INTEGER REFERENCES orgs(id) ON DELETE CASCADE
     )""")
     cursor.execute("""
-    CREATE INDEX IF NOT EXISTS org_news_index ON org_news (author, recipient, subject, timestamp)
+    CREATE INDEX IF NOT EXISTS org_news_index ON org_news (newsid, author, recipient, subject, timestamp, orgid)
     """)
     cursor.execute("""
     CREATE VIRTUAL TABLE IF NOT EXISTS org_news_fts USING fts4(content="org_news", f_content)
 
     try:
         total_objects = len(results)
+        orgname = results[0][0]
+        orgid = ProcessOrg(orgname)
+        import_set = []
         for i in range(len(results)):
-            (orgname, newsid, timestamp, author, recipient, subject, f_content) = results[i]
-            ProcessNews(orgname, newsid, timestamp, author, recipient, subject, f_content)
+            (newsid, timestamp, author, recipient, subject, f_content) = results[i][1:]
+            cursor.execute("SELECT newsid FROM org_news WHERE orgid=? AND newsid=?", (orgname, newsid))
+            if not len(results) > 0: # here we prune results to avoid duplicates
+                import_set.append(newsid, author, recipient, subject, timestamp, f_content, orgid)
+        cursor.executemany("""
+        INSERT INTO org_news (newsid, author, recipient, subject, timestamp, f_content, orgid) VALUES (?, ?, ?, ?, datetime(?), ?, ?)
+        """, import_set) # this line makes the inserts go super fast
         Note("NewsDB: Imported a total of %d objects." % total_objects)
+    except sqlite3.Error:
+        Note("SQLite error: {0}".format(e.message))
     except:
         Note("NewsDB: Invalid file or the file does not contain a NewsDB pickle object.")