Commits

Anonymous committed a80605e

Completada versión 2

Comments (0)

Files changed (3)

 
 
 # Leer parámetros de búsqueda
-from util import openutf8, openlat1
+from database import materias_dict, editoriales_dict
 
-MATERIAS = dict(r[:-1].split("\t", 1)
-                for r in openutf8("materias.txt") if r != "\n")
-EDITORIALES = {
-    unicode(k): [t for t in s.split("\t") if t]
-    for (k, s) in
-        (r[:-1].split("\t", 1)
-         for r in openlat1("editoriales2.txt") if r != "\n")
-}
+MATERIAS = materias_dict()
+EDITORIALES = editoriales_dict()
 DB = sqlite3.connect(DB_FILE,)
 
 INSERTS = {
+    "materias": "insert into materias(ibic, desc) values(?,?)",
+    "editoriales": "insert into editoriales(nombre, codigos) values(?,?)",
     "libros": """insert into libros(ISBN,
                                     idbook,
                                     título,
 
 def init_db():
 
-    from config import MATERIAS, EDITORIALES
-
     with DB:
 
         DB.execute("drop table IF EXISTS libros")
 
         DB.execute("""create table materias (ibic varchar(5) PRIMARY KEY,
                                              desc text)""")
-        DB.executemany("insert into materias values(?,?)",
-                       MATERIAS.iteritems())
 
         DB.execute("""create table editoriales (id varchar(5),
                                                 nombre varchar(60),
                                                 codigos text)""")
-        DB.executemany("insert into editoriales(nombre, codigos) values(?,?)",
-                       ((nombre, repr(codes))
-                        for (nombre, codes) in EDITORIALES.iteritems()))
-
         DB.execute("""create table libros(
                             ISBN text,
                             idbook text,
 
         DB.execute("""create table logs(
                             fecha timestamp DEFAULT CURRENT_TIMESTAMP,
+                            parm1 varchar(10),
+                            parm2 varchar(10),
                             desc  text)""")
 
         log("Tablas creadas")
 
 
-def log(*msg):
+def fill_materias(fname):
+    from util import openutf8
+
+    materias = (r[:-1].split("\t", 1)
+                for r in openutf8(fname) if r != "\n")
+
+    query = INSERTS["materias"]
+
     with DB:
-        DB.execute("insert into logs(desc) values(?)", [" ".join(msg)])
+        DB.executemany(query, materias)
+
+    log("Tabla materias llenada del fichero", fname)
+
+
+def materias_dict():
+    query = "select ibic, desc from materias"
+    return dict(DB.execute(query))
+
+
+def fill_editoriales(fname):
+    import re
+    from util import openlat1
+
+    isbn = re.compile(r"([-\d]+)")
+
+    editoriales = ((k, ",".join(isbn.findall(s)))
+                   for (k, s) in (r.split("\t", 1)
+                                  for r in openlat1(fname)))
+
+    query = INSERTS["editoriales"]
+
+    with DB:
+        DB.executemany(query, editoriales)
+
+    log("Tabla editoriales llenada del fichero", fname)
+
+
+def editoriales_dict():
+    query = "select nombre, codigos from editoriales"
+    return dict((k, v.split(",")) for (k, v) in DB.execute(query))
+
+
+class Logger(object):
+
+    def __init__(self):
+        self.parm1 = ""
+        self.parm2 = ""
+
+    def set_parms(self, parm1, parm2):
+        self.parm1 = parm1
+        self.parm2 = parm2
+
+    def __call__(self, *msg):
+
+        with DB:
+            DB.execute("insert into logs(parm1, parm2, desc) values(?,?,?)",
+                       [self.parm1, self.parm2, " ".join(msg)])
+
+log = Logger()
+
+
+if __name__ == "__main__":
+
+    init_db()
+    fill_materias("materias.txt")
+    fill_editoriales("editoriales2.txt")
+
+    print materias_dict()
+    print editoriales_dict()
 __updated__ = "2014-04-14 19:10"
 
 import time
+from random import random
 
 from mk_query import QA_IBIC_ISBN
 from get_isbn import get_books
                 query = QA_IBIC_ISBN(materia, isbn, YEAR1, YEAR2,
                                      maxitems=MAXITEMS,
                                      publanguage=PUBLANG)
-                msg = "IBIC:{} ISBN:{} Editorial:{}".format(materia,
-                                                            isbn,
-                                                            editorial)
-                log(msg)
+
+                log.set_parms(materia, isbn)
+                log("Editorial", editorial)
 
                 for book in get_books(query, logger=log):
                     data = book + [materia, m_desc]
                 DB.commit()
                 time.sleep(2)  # Espera de 2 segundos entre peticiones
 
+        time.sleep((1+2*random())*60)  # Espera entre 1 y 3 minutos
+
     DB.commit()