Commits

Marko Toplak committed 6e88615

Fixed update scripts for MeSH, GO, HomoloGene, NCBI_geneinfo, OMIM, PPI. Moved wget to Orange.utils.

  • Participants
  • Parent commits 146dc04

Comments (0)

Files changed (11)

File _bioinformatics/obiGeneMania.py

     except Exception, ex:
         pass
 
-
-def copyfileobj(src, dst, buffer=2**10, content_len=None, progress=None):
-    count = 0
-    if content_len is None:
-        content_len = guess_size(src) or sys.maxint
-    while True:
-        data = src.read(buffer)
-        dst.write(data)
-        count += len(data)
-        if progress:
-            progress(100.0 * count / content_len)
-        if not data:
-            break
-            
-            
-def wget(url, directory=".", dst_obj=None, progress=None):
-    """
-    .. todo:: Move to Orange.misc
-    
-    """
-    stream = urllib2.urlopen(url)
-    length = stream.headers.get("content-length", None)
-    if length is None:
-        length = sys.maxint
-    else:
-        length = int(length)
-    
-    basename = posixpath.basename(url)
-        
-    if dst_obj is None:
-        dst_obj = open(os.path.join(directory, basename), "wb")
-    
-    if progress == True:
-        from Orange.utils import ConsoleProgressBar
-        progress = ConsoleProgressBar("Downloading %r." % basename)
-        with finishing(progress):
-            copyfileobj(stream, dst_obj, buffer=2**10, content_len=length,
-                        progress=progress)
-    else:
-        copyfileobj(stream, dst_obj, buffer=2**10, content_len=length,
-                    progress=progress)
+from Orange.utils import wget
     
 from . import obiPPI
 from Orange.orng import orngServerFiles

File _bioinformatics/obiGeneSets.py

 """
-Getting genesets from KEGG and GO.
-
 Maintainer: Marko Toplak
 """
 

File _bioinformatics/obiOMIM.py

             if not os.path.exists(cls.DEFAULT_DATABASE_PATH):
                 os.mkdir(cls.DEFAULT_DATABASE_PATH)
             file = open(os.path.join(cls.DEFAULT_DATABASE_PATH, "morbidmap"), "wb")
-        elif type(file) in [str, unicode]:
+        elif isinstance(file, basestring):
             file = open(file, "wb")
         file.write(data)
+        file.close()
         
     @classmethod
     def get_instance(cls):

File _bioinformatics/obiPPI.py

 from operator import itemgetter
 
 from Orange.orng import orngServerFiles
-from Orange.utils import ConsoleProgressBar, lru_cache
+from Orange.utils import ConsoleProgressBar, lru_cache, wget
 
 from . import obiTaxonomy
 
         primary ids. Use `taxid` to limit the results to a single organism.
          
         """
-        # TODO: synonyms_interactor can contain multiple synonyms
-        # (should create an indexed table of synonyms)
+            # TODO: synonyms_interactor can contain multiple synonyms
+            # (should create an indexed table of synonyms)
         if taxid is None:
             cur = self.db.execute("""\
                 select biogrid_id_interactor
                     orngServerFiles.localpath("PPI", "BIOGRID-ALL.tab2"))
         filepath = orngServerFiles.localpath("PPI", "BIOGRID-ALL.tab2")
         cls.init_db(filepath)
-        shutil.remove(filepath)
+        os.remove(filepath)
         
     @classmethod
     def init_db(cls, filepath):
         lineiter = iter(open(filepath, "rb"))
         headers = lineiter.next() # read the first line
         
-        con = sqlite3.connect(os.path.join(dirname, "BIOGRID-ALL.sqlite"))
+        con = sqlite3.connect(os.path.join(dirname, BioGRID.SERVER_FILE))
         con.execute("drop table if exists links") # Drop old table
         con.execute("drop table if exists proteins") # Drop old table
         
 from collections import namedtuple
 from functools import partial
 
-from .obiGeneMania import wget, copyfileobj
-
 def chainiter(iterable):
     for sub_iter in iterable:
         for obj in sub_iter:

File server_update/updateGO.py

 ##!interval=7
 ##!contact=ales.erjavec@fri.uni-lj.si
 
-import obiGO, obiTaxonomy, obiGene, obiGenomicsUpdate, orngEnviron, orngServerFiles
+from Orange.bio import obiGO, obiTaxonomy, obiGene, obiGenomicsUpdate
+
+import Orange.utils.environ as orngEnviron
+import Orange.utils.serverfiles as orngServerFiles
 import os, sys, shutil, urllib2, tarfile
 from getopt import getopt
 
 
 from collections import defaultdict
 
-tmpDir = os.path.join(orngEnviron.bufferDir, "tmp_GO")
+tmpDir = os.path.join(orngEnviron.buffer_dir, "tmp_GO")
 try:
     os.mkdir(tmpDir)
 except Exception:
 exclude = ["goa_uniprot", "goa_pdb", "GeneDB_tsetse", "reactome", "goa_zebrafish", "goa_rat", "goa_mouse"]
 
 updatedTaxonomy = defaultdict(set)
-import obiTaxonomy
 
 for org in u.GetAvailableOrganisms():
     if org in exclude or org not in commonOrgs:
     print "Uploading", "taxonomy.pickle"
     serverFiles.upload("GO", "taxonomy.pickle", os.path.join(tmpDir, "taxonomy.pickle"), title="GO taxon IDs",
                        tags = ["GO", "taxon", "organism", "essential", "#version:%i" % obiGO.Taxonomy.version])
-    serverFiles.unprotect("GO", "taxonomy.pickle")
+    serverFiles.unprotect("GO", "taxonomy.pickle")

File server_update/updateHomoloGene.py

 ##!interval=7
 ##!contact=ales.erjavec@fri.uni-lj.si
 
-import obiHomoloGene
-import orngServerFiles
+from Orange.bio import obiHomoloGene
+import Orange.utils.serverfiles as orngServerFiles
 
-import orngEnviron
+import Orange.utils.environ as orngEnviron
 import os, sys
 import gzip, shutil
 
 username = opt.get("-u", opt.get("--user", "username"))
 password = opt.get("-p", opt.get("--password", "password"))
 
-path = os.path.join(orngEnviron.bufferDir, "tmp_HomoloGene")
+path = os.path.join(orngEnviron.buffer_dir, "tmp_HomoloGene")
 serverFiles = orngServerFiles.ServerFiles(username, password)
 
 try:
 serverFiles.unprotect("HomoloGene", "InParanoid.sqlite")
         
         
-            
+            

File server_update/updateMeSH.py

 ##!contact=ales.erjavec@fri.uni-lj.si
 
 from urllib import urlopen
-import orngServerFiles
+import Orange.utils.serverfiles as orngServerFiles
 import os, sys
 
 from getopt import getopt
 password = opt.get("-p", opt.get("--password", "password"))
 
 
-ontology = urlopen("ftp://nlmpubs.nlm.nih.gov/online/mesh/.asciimesh/d2008.bin")
+ontology = urlopen("ftp://nlmpubs.nlm.nih.gov/online/mesh/.asciimesh/d2012.bin")
 size = int(ontology.info().getheader("Content-Length"))
 rsize = 0
 results = list()
 
 authenticated.unprotect('MeSH', 'mesh-ontology.dat')
 os.remove('mesh-ontology.dat')
-print "Ontology uploaded to server."
+print "Ontology uploaded to server."

File server_update/updateNCBI_geneinfo.py

 ##interval:7
-import obiGene, obiTaxonomy
-import orngServerFiles, orngEnviron
+from Orange.bio import obiGene, obiTaxonomy
+import Orange.utils.serverfiles as orngServerFiles
+import Orange.utils.environ as orngEnviron
 import sys, os
 from gzip import GzipFile
 from getopt import getopt
 
 opt = dict(getopt(sys.argv[1:], "u:p:", ["user=", "password="])[0])
 
-tmpdir = os.path.join(orngEnviron.bufferDir, "tmp_NCBIGene_info")
+tmpdir = os.path.join(orngEnviron.buffer_dir, "tmp_NCBIGene_info")
 try:
     os.mkdir(tmpdir)
 except Exception, ex:

File server_update/updateOMIM.py

 ##!interval=7
 ##!contact=ales.erjavec@fri.uni-lj.si
 
-import obiOMIM
-import orngServerFiles
+from Orange.bio import obiOMIM
+import Orange.utils.serverfiles as orngServerFiles
 
 import orngEnviron
 import os, sys

File server_update/updatePPI.py

 ##!interval=7
 ##!contact=ales.erjavec@fri.uni-lj.si
 
-import obiPPI, orngServerFiles
+from Orange.bio import obiPPI
+import Orange.utils.serverfiles as orngServerFiles
 import os, sys, shutil, urllib2, tarfile
 from getopt import getopt
 
 except OSError:
     pass
 
-obiPPI.MIPS.download()
-
 try:
     serverFiles.create_domain("PPI")
 except Exception, ex:
     print ex
-filename = orngServerFiles.localpath("PPI", "mppi.gz")
-serverFiles.upload("PPI", "allppis.xml", filename, "MIPS Protein interactions",
-                   tags=["protein interaction", "MIPS", "#compression:gz", "#version:%i" % obiPPI.MIPS.VERSION]
-                   )
-serverFiles.unprotect("PPI", "allppis.xml") 
 
-if False: ## download BIOGRID-ALL manually
+if True:
+    obiPPI.MIPS.download()
+
+    filename = orngServerFiles.localpath("PPI", "mppi.gz")
+    serverFiles.upload("PPI", "allppis.xml", filename, "MIPS Protein interactions",
+                       tags=["protein interaction", "MIPS", "#compression:gz", "#version:%i" % obiPPI.MIPS.VERSION]
+                       )
+    serverFiles.unprotect("PPI", "allppis.xml") 
+
+if True:
+    obiPPI.BioGRID.download_data("http://thebiogrid.org/downloads/archives/Release%20Archive/BIOGRID-3.1.91/BIOGRID-ALL-3.1.91.tab2.zip") #replace with the newest version
+
+    sfn = obiPPI.BioGRID.SERVER_FILE
+
+    filename = orngServerFiles.localpath("PPI", sfn)
+
     import gzip
-    filename = orngServerFiles.localpath("PPI", "BIOGRID-ALL.tab")
     gz = gzip.GzipFile(filename + ".gz", "wb")
     gz.write(open(filename, "rb").read())
     gz.close()
-    serverFiles.upload("PPI", "BIOGRID-ALL.tab", filename + ".gz", title="BioGRID Protein interactions", 
-                       tags=["protein interaction", "BioGrid", "#compression:gz", "#version:%i" % obiPPI.BioGRID.VERSION]
-                       )
-    serverFiles.unprotect("PPI", "BIOGRID-ALL.tab")
 
+    serverFiles.upload("PPI", sfn, filename + ".gz", 
+        title="BioGRID Protein interactions", 
+        tags=["protein interaction", "BioGrid", "#compression:gz", "#version:%s" % obiPPI.BioGRID.VERSION]
+        )
+    serverFiles.unprotect("PPI", sfn)
+

File server_update/updatemiRNA.py

 from getopt import getopt
 import zipfile
 
-import obiTaxonomy as tax
-import orngServerFiles
-import orngEnviron
+import Orange.bio.obiTaxonomy as tax
+import Orange.utils.serverfiles as orngServerFiles
+import Orange.utils.environ as orngEnviron
 
 def fastprint(filename,mode,what):
     
 
 def sendMail(subject):
     
+    print "MAIL", subject
     toaddr = "rsberex@yahoo.it"
     fromaddr = "orange@fri.uni-lj.si";
     msg = "From: %s\r\nTo: %s\r\nSubject: %s" % (fromaddr, toaddr, subject)
 username = opt.get("-u", opt.get("--user", "username"))
 password = opt.get("-p", opt.get("--password", "password"))
 
-path = os.path.join(orngEnviron.bufferDir, "tmp_miRNA")
+path = os.path.join(orngEnviron.buffer_dir, "tmp_miRNA")
 print 'path: ', path
 
 serverFiles = orngServerFiles.ServerFiles(username, password)