1. okfn
  2. jiscobib

Commits

w...@styx.org  committed 8f32fbc

checkpoint after each file... improve fast loader...

  • Participants
  • Parent commits 2800fb8
  • Branches default

Comments (0)

Files changed (1)

File jiscobib/blstore.py

View file
     cursor = self.handler.rdflib.store.cursor()
 
     for fname in files_to_convert:
+      base = os.path.basename(fname)
       log.info("Processing %s" % fname)
       file_no = get_file_number(fname)
       record = 1
           rdf = head + \
               u'\n    <rdf:Description rdf:about="%s">\n' % ident + \
               u'\n'.join(lines)
-          self.handler.rdflib.store.query(
           q = u"""\
 DB.DBA.RDF_LOAD_RDFXML('%s', '', '%s')\
-""" % (rdf.replace("'", "\\'"), ident))
+""" % (rdf.replace("'", "\\'"), ident)
 	  cursor.execute(q.encode("utf-8"))
           if record % 1000 == 0:
             q = u"COMMIT WORK"
 
 #          self.handler.put(g)
 #          print g.serialize(format='n3')
+
+          log.info("Stored %s:%s: %s" % (base, record, ident))
+#          q = u"COMMIT WORK"
+#          cursor.execute(q.encode("utf-8"))
 #          sys.exit(0)
-
-          log.info("Stored %s (%s)" % (b_uri, bnb_id))
-          lines = [head]
+          lines = []
           record = record + 1
           bnb_id = None
         elif line in [u"    <dcterms:contributor>",
 
       q = u"COMMIT WORK"
       cursor.execute(q.encode("utf-8"))
+      q = u"CHECKPOINT"
+      cursor.execute(q.encode("utf-8"))
 
 def convert_and_store():
   ConvertAndStore().command()