Anonymous avatar Anonymous committed 8f32fbc

checkpoint after each file... improve fast loader...

Comments (0)

Files changed (1)

jiscobib/blstore.py

     cursor = self.handler.rdflib.store.cursor()
 
     for fname in files_to_convert:
+      base = os.path.basename(fname)
       log.info("Processing %s" % fname)
       file_no = get_file_number(fname)
       record = 1
           rdf = head + \
               u'\n    <rdf:Description rdf:about="%s">\n' % ident + \
               u'\n'.join(lines)
-          self.handler.rdflib.store.query(
           q = u"""\
 DB.DBA.RDF_LOAD_RDFXML('%s', '', '%s')\
-""" % (rdf.replace("'", "\\'"), ident))
+""" % (rdf.replace("'", "\\'"), ident)
 	  cursor.execute(q.encode("utf-8"))
           if record % 1000 == 0:
             q = u"COMMIT WORK"
 
 #          self.handler.put(g)
 #          print g.serialize(format='n3')
+
+          log.info("Stored %s:%s: %s" % (base, record, ident))
+#          q = u"COMMIT WORK"
+#          cursor.execute(q.encode("utf-8"))
 #          sys.exit(0)
-
-          log.info("Stored %s (%s)" % (b_uri, bnb_id))
-          lines = [head]
+          lines = []
           record = record + 1
           bnb_id = None
         elif line in [u"    <dcterms:contributor>",
 
       q = u"COMMIT WORK"
       cursor.execute(q.encode("utf-8"))
+      q = u"CHECKPOINT"
+      cursor.execute(q.encode("utf-8"))
 
 def convert_and_store():
   ConvertAndStore().command()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.