Commits

Matthew Shirley  committed 0fe8cc9 Draft

modified data_source.py to accept an XML file that defines multiple resources to download

  • Participants
  • Parent commits 559effe

Comments (0)

Files changed (1)

File tools/data_source/data_source.py

 #!/usr/bin/env python
 # Retrieves data from external data source applications and stores in a dataset file.
 # Data source application parameters are temporarily stored in the dataset file.
-import socket, urllib, sys, os
+import socket, urllib, sys, os, optparse
+import xml.etree.ElementTree as ElementTree
 from galaxy import eggs #eggs needs to be imported so that galaxy.util can find docutils egg...
 from galaxy.util.json import from_json_string, to_json_string
 from galaxy.util import get_charset_from_http_headers
         open( filename, 'w' ).close() #open file for writing, then close, removes params from file
     return json_params, datasource_params
 
+def deconstruct_multi_filename( multi_filename ):
+    keys = ['primary', 'id', 'name', 'visible', 'file_type', 'dbkey']
+    return ( dict( zip( keys, multi_filename.split('_') ) ) )
+    
+def construct_multi_filename( id, name, file_type, dbkey=None):
+    """ Implementation of *Number of Output datasets cannot be determined until tool run* from documentation_.
+    .. _documentation: http://wiki.galaxyproject.org/Admin/Tools/Multiple%20Output%20Files
+    """
+    if dbkey:
+        filename = "%s_%s_%s_%s_%s_%s" % ( 'primary', id, name, 'visible', file_type, dbkey )
+    else:
+        filename = "%s_%s_%s_%s_%s" % ( 'primary', id, name, 'visible', file_type )
+        
+    return filename
+    
+def xml_save_to_newfile_directory( xmlfile, directory, id, enhanced_handling=False, datatypes_registry=None ):
+    """ Open xmlfile, parse all URLs to fetch. Fetch each file, saving to ``directory``. 
+    Save first file for last and return for ``page``. 
+    
+    Schema
+    
+    ::
+    
+        <?xml version="1.0"?>
+        <!DOCTYPE downloads [
+        <!ELEMENT downloads (download)>
+        <!ELEMENT download (resource,url,meta)>
+        <!ELEMENT resource (#PCDATA)>
+        <!ELEMENT url (#PCDATA)>
+        <!ELEMENT id (#PCDATA)>
+        <!ELEMENT meta (id,format,type,summary,feature,genome,technique,instrument,assay,sample,description,PMID)>
+        <!ELEMENT id (#PCDATA)>
+        <!ELEMENT format (#PCDATA)>
+        <!ELEMENT type (#PCDATA)>
+        <!ELEMENT summary (#PCDATA)>
+        <!ELEMENT feature (#PCDATA)>
+        <!ELEMENT genome (#PCDATA)>
+        <!ELEMENT technique (#PCDATA)>
+        <!ELEMENT instrument (#PCDATA)>
+        <!ELEMENT assay (#PCDATA)>
+        <!ELEMENT sample (#PCDATA)>
+        <!ELEMENT description (#PCDATA)>
+        <!ELEMENT PMID (#PCDATA)>
+        ]>
+    """
+    root = ElementTree.fromstring(xmlfile.read())
+    if root.tag != 'downloads':
+        stop_err( 'The remote data source application has not sent back a URL parameter in the request.' )
+    # traverse xml schema to find URLs, names, and dbkeys
+    files_to_fetch = []
+    complete = False
+    
+    for child in root:
+      if (child.tag == 'download') and (complete == True):
+        files_to_fetch.append( ( construct_multi_filename( id, name, file_type, dbkey ), URL ) )
+        
+      for sub in child:
+        if sub.tag == 'url':
+          URL = sub.text
+        elif sub.tag == 'meta':
+        
+          for meta in sub:
+            if meta.tag == 'id':
+              name = meta.text
+            elif meta.tag == 'genome':
+              dbkey = meta.text
+            elif meta.tag == 'format':
+              file_type = meta.text
+          # hit the end of our schema    
+          files_to_fetch.append( ( construct_multi_filename( id, name, file_type, dbkey ), URL ) )
+    
+    if len(files_to_fetch) > 1:
+        for filename, URL in files_to_fetch[1:]:
+            try:
+                cur_filename = os.path.join( directory, filename )
+                page = urllib.urlopen( URL )
+                multi_dict = deconstruct_multi_filename( filename )
+                cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename, source_encoding=get_charset_from_http_headers( page.headers ) )
+                if enhanced_handling:
+                    ext = sniff.handle_uploaded_dataset_file( cur_filename, datatypes_registry, ext = multi_dict['file_type'], is_multi_byte = is_multi_byte )
+            except Exception, e:
+                stop_err( 'Unable to fetch %s:\n%s' % ( URL, e ) )
+    
+    # pass page back to main
+    return ( files_to_fetch[0][0], urllib.urlopen( files_to_fetch[0][1] ) )
+
 def __main__():
-    filename = sys.argv[1]
+    # Parse the command line options
+    usage = "Usage: data_source.py filename max_size [options]"
+    parser = optparse.OptionParser(usage = usage)
+    parser.add_option("-x", "--xml",
+                    action="store_true", dest="xmlfile", help="filename defines external resources as xml")
+    parser.add_option("-i", "--id", type="string",
+                    action="store", dest="id", help="output id")
+    parser.add_option("-p", "--path", type="string",
+                    action="store", dest="newfilepath", help="new file path")
+                    
+    (options, args) = parser.parse_args()
+  
+    filename = args[0]
     try:
-        max_file_size = int( sys.argv[2] )
+        max_file_size = int( args[1] )
     except:
         max_file_size = 0
     
             file_size = int( page.info().get( 'Content-Length', 0 ) )
             if file_size > max_file_size:
                 stop_err( 'The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % ( file_size, max_file_size ) )
+        # If xmlfile is provided, fetch files 2 through n and save to new_file_path, replace page with file 1
+        if options.xmlfile:
+            multi_filename, page = xml_save_to_newfile_directory( page, options.newfilepath, options.id, enhanced_handling, datatypes_registry )
+            multi_dict = deconstruct_multi_filename( multi_filename )
+            # replace metadata elements with info from xmlfile
+            
         #do sniff stream for multi_byte
         try:
             cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename, source_encoding=get_charset_from_http_headers( page.headers ) )