Commits

Brad Chapman committed 8103ef1

Correct upload.py problem from last merge. Thanks to Gianmauro Cuccuru

Comments (0)

Files changed (1)

tools/data_source/upload.py

+#!/usr/bin/env python
+#Processes uploads from the user.
 
+# WARNING: Changes in this tool (particularly as related to parsing) may need
+# to be reflected in galaxy.web.controllers.tool_runner and galaxy.tools
 
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <title>
-  galaxy / galaxy-central / source &mdash; Bitbucket
-</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-  <meta name="description" content="" />
-  <meta name="keywords" content="" />
-  
-    
-  
-  <!--[if lt IE 9]>
-  <script src="https://dwz7u9t8u8usb.cloudfront.net/m/33b98326237f/js/old/html5.js"></script>
-  <![endif]-->
+import urllib, sys, os, gzip, tempfile, shutil, re, gzip, zipfile, codecs, binascii
+from galaxy import eggs
+# need to import model before sniff to resolve a circular import dependency
+import galaxy.model
+from galaxy.datatypes.checkers import *
+from galaxy.datatypes import sniff
+from galaxy.datatypes.binary import *
+from galaxy.datatypes.images import Pdf
+from galaxy.datatypes.registry import Registry
+from galaxy import util
+from galaxy.datatypes.util.image_util import *
+from galaxy.util.json import *
 
-  <script>
-    (function (window) {
-      // prevent stray occurrences of `console.log` from causing errors in IE
-      var console = window.console || (window.console = {});
-      console.log || (console.log = function () {});
+try:
+    import Image as PIL
+except ImportError:
+    try:
+        from PIL import Image as PIL
+    except:
+        PIL = None
 
-      var BB = window.BB || (window.BB = {});
-      BB.debug = false;
-      BB.cname = false;
-      BB.CANON_URL = 'https://bitbucket.org';
-      BB.STATIC_URL = 'https://dwz7u9t8u8usb.cloudfront.net/m/33b98326237f/';
-      BB.images = {
-        invitation: 'https://dwz7u9t8u8usb.cloudfront.net/m/33b98326237f/img/icons/fugue/card_address.png',
-        noAvatar: 'https://dwz7u9t8u8usb.cloudfront.net/m/33b98326237f/img/default_avatar/16/user_blue.png'
-      };
-      BB.user = {"isKbdShortcutsEnabled": true, "isSshEnabled": false, "isAuthenticated": false};
-      BB.targetUser = BB.user;
-    
-      BB.repo || (BB.repo = {});
-  
-      BB.repo.id = 6304;
-    
-    
-      BB.repo.language = null;
-      BB.repo.pygmentsLanguage = null;
-    
-    
-      BB.repo.slug = 'galaxy\u002Dcentral';
-    
-    
-      BB.repo.owner = {};
-      BB.repo.owner.username = 'galaxy';
-    
-    
-      // Coerce `BB.repo` to a string to get
-      // "davidchambers/mango" or whatever.
-      BB.repo.toString = function () {
-        return BB.cname ? this.slug : '{owner.username}/{slug}'.format(this);
-      }
-    
-      BB.changeset = '05fc04a70a3bcbfaeedfcf6f2ec16a7e38fc7c94'
-    
-    
-  
-    }(this));
-  </script>
+try:
+    import bz2
+except:
+    bz2 = None
 
-  
-<link rel="stylesheet" href="https://dwz7u9t8u8usb.cloudfront.net/m/33b98326237f/compressed/css/2a9f6ed1880e.css" type="text/css" />
+assert sys.version_info[:2] >= ( 2, 4 )
 
-<link rel="stylesheet" href="https://dwz7u9t8u8usb.cloudfront.net/m/33b98326237f/compressed/css/ba82c2292cc6.css" type="text/css" />
+def stop_err( msg, ret=1 ):
+    sys.stderr.write( msg )
+    sys.exit( ret )
+def file_err( msg, dataset, json_file ):
+    json_file.write( to_json_string( dict( type = 'dataset',
+                                           ext = 'data',
+                                           dataset_id = dataset.dataset_id,
+                                           stderr = msg ) ) + "\n" )
+    # never remove a server-side upload
+    if dataset.type in ( 'server_dir', 'path_paste' ):
+        return
+    try:
+        os.remove( dataset.path )
+    except:
+        pass
+def safe_dict(d):
+    """
+    Recursively clone json structure with UTF-8 dictionary keys
+    http://mellowmachines.com/blog/2009/06/exploding-dictionary-with-unicode-keys-as-python-arguments/
+    """
+    if isinstance(d, dict):
+        return dict([(k.encode('utf-8'), safe_dict(v)) for k,v in d.iteritems()])
+    elif isinstance(d, list):
+        return [safe_dict(x) for x in d]
+    else:
+        return d
+def parse_outputs( args ):
+    rval = {}
+    for arg in args:
+        id, files_path, path = arg.split( ':', 2 )
+        rval[int( id )] = ( path, files_path )
+    return rval
+def add_file( dataset, registry, json_file, output_path ):
+    data_type = None
+    line_count = None
+    converted_path = None
+    stdout = None
+    link_data_only = dataset.get( 'link_data_only', 'copy_files' )
+    in_place = dataset.get( 'in_place', True )
 
+    try:
+        ext = dataset.file_type
+    except AttributeError:
+        file_err( 'Unable to process uploaded file, missing file_type parameter.', dataset, json_file )
+        return
 
-  <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="Bitbucket" />
-  <link rel="icon" href="https://dwz7u9t8u8usb.cloudfront.net/m/33b98326237f/img/logo_new.png" type="image/png" />
-  <link type="text/plain" rel="author" href="/humans.txt" />
+    if dataset.type == 'url':
+        try:
+            page = urllib.urlopen( dataset.path ) #page will be .close()ed by sniff methods
+            temp_name, dataset.is_multi_byte = sniff.stream_to_file( page, prefix='url_paste', source_encoding=util.get_charset_from_http_headers( page.headers ) )
+        except Exception, e:
+            file_err( 'Unable to fetch %s\n%s' % ( dataset.path, str( e ) ), dataset, json_file )
+            return
+        dataset.path = temp_name
+    # See if we have an empty file
+    if not os.path.exists( dataset.path ):
+        file_err( 'Uploaded temporary file (%s) does not exist.' % dataset.path, dataset, json_file )
+        return
+    if not os.path.getsize( dataset.path ) > 0:
+        file_err( 'The uploaded file is empty', dataset, json_file )
+        return
+    if not dataset.type == 'url':
+        # Already set is_multi_byte above if type == 'url'
+        try:
+            dataset.is_multi_byte = util.is_multi_byte( codecs.open( dataset.path, 'r', 'utf-8' ).read( 100 ) )
+        except UnicodeDecodeError, e:
+            dataset.is_multi_byte = False
+    # Is dataset an image?
+    image = check_image( dataset.path )
+    if image:
+        if not PIL:
+            image = None
+        # get_image_ext() returns None if nor a supported Image type
+        ext = get_image_ext( dataset.path, image )
+        data_type = ext
+    # Is dataset content multi-byte?
+    elif dataset.is_multi_byte:
+        data_type = 'multi-byte char'
+        ext = sniff.guess_ext( dataset.path, is_multi_byte=True )
+    # Is dataset content supported sniffable binary?
+    else:
+        type_info = Binary.is_sniffable_binary( dataset.path )
+        if type_info:
+            data_type = type_info[0]
+            ext = type_info[1]
+    if not data_type:
+        # See if we have a gzipped file, which, if it passes our restrictions, we'll uncompress
+        is_gzipped, is_valid = check_gzip( dataset.path )
+        if is_gzipped and not is_valid:
+            file_err( 'The gzipped uploaded file contains inappropriate content', dataset, json_file )
+            return
+        elif is_gzipped and is_valid:
+            if link_data_only == 'copy_files':
+                # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format
+                CHUNK_SIZE = 2**20 # 1Mb   
+                fd, uncompressed = tempfile.mkstemp( prefix='data_id_%s_upload_gunzip_' % dataset.dataset_id, dir=os.path.dirname( output_path ), text=False )
+                gzipped_file = gzip.GzipFile( dataset.path, 'rb' )
+                while 1:
+                    try:
+                        chunk = gzipped_file.read( CHUNK_SIZE )
+                    except IOError:
+                        os.close( fd )
+                        os.remove( uncompressed )
+                        file_err( 'Problem decompressing gzipped data', dataset, json_file )
+                        return
+                    if not chunk:
+                        break
+                    os.write( fd, chunk )
+                os.close( fd )
+                gzipped_file.close()
+                # Replace the gzipped file with the decompressed file if it's safe to do so
+                if dataset.type in ( 'server_dir', 'path_paste' ) or not in_place:
+                    dataset.path = uncompressed
+                else:
+                    shutil.move( uncompressed, dataset.path )
+                os.chmod(dataset.path, 0644)
+            dataset.name = dataset.name.rstrip( '.gz' )
+            data_type = 'gzip'
+        if not data_type and bz2 is not None:
+            # See if we have a bz2 file, much like gzip
+            is_bzipped, is_valid = check_bz2( dataset.path )
+            if is_bzipped and not is_valid:
+                file_err( 'The gzipped uploaded file contains inappropriate content', dataset, json_file )
+                return
+            elif is_bzipped and is_valid:
+                if link_data_only == 'copy_files':
+                    # We need to uncompress the temp_name file
+                    CHUNK_SIZE = 2**20 # 1Mb   
+                    fd, uncompressed = tempfile.mkstemp( prefix='data_id_%s_upload_bunzip2_' % dataset.dataset_id, dir=os.path.dirname( output_path ), text=False )
+                    bzipped_file = bz2.BZ2File( dataset.path, 'rb' )
+                    while 1:
+                        try:
+                            chunk = bzipped_file.read( CHUNK_SIZE )
+                        except IOError:
+                            os.close( fd )
+                            os.remove( uncompressed )
+                            file_err( 'Problem decompressing bz2 compressed data', dataset, json_file )
+                            return
+                        if not chunk:
+                            break
+                        os.write( fd, chunk )
+                    os.close( fd )
+                    bzipped_file.close()
+                    # Replace the bzipped file with the decompressed file if it's safe to do so
+                    if dataset.type in ( 'server_dir', 'path_paste' ) or not in_place:
+                        dataset.path = uncompressed
+                    else:
+                        shutil.move( uncompressed, dataset.path )
+                    os.chmod(dataset.path, 0644)
+                dataset.name = dataset.name.rstrip( '.bz2' )
+                data_type = 'bz2'
+        if not data_type:
+            # See if we have a zip archive
+            is_zipped = check_zip( dataset.path )
+            if is_zipped:
+                if link_data_only == 'copy_files':
+                    CHUNK_SIZE = 2**20 # 1Mb
+                    uncompressed = None
+                    uncompressed_name = None
+                    unzipped = False
+                    z = zipfile.ZipFile( dataset.path )
+                    for name in z.namelist():
+                        if name.endswith('/'):
+                            continue
+                        if unzipped:
+                            stdout = 'ZIP file contained more than one file, only the first file was added to Galaxy.'
+                            break
+                        fd, uncompressed = tempfile.mkstemp( prefix='data_id_%s_upload_zip_' % dataset.dataset_id, dir=os.path.dirname( output_path ), text=False )
+                        if sys.version_info[:2] >= ( 2, 6 ):
+                            zipped_file = z.open( name )
+                            while 1:
+                                try:
+                                    chunk = zipped_file.read( CHUNK_SIZE )
+                                except IOError:
+                                    os.close( fd )
+                                    os.remove( uncompressed )
+                                    file_err( 'Problem decompressing zipped data', dataset, json_file )
+                                    return
+                                if not chunk:
+                                    break
+                                os.write( fd, chunk )
+                            os.close( fd )
+                            zipped_file.close()
+                            uncompressed_name = name
+                            unzipped = True
+                        else:
+                            # python < 2.5 doesn't have a way to read members in chunks(!)
+                            try:
+                                outfile = open( uncompressed, 'wb' )
+                                outfile.write( z.read( name ) )
+                                outfile.close()
+                                uncompressed_name = name
+                                unzipped = True
+                            except IOError:
+                                os.close( fd )
+                                os.remove( uncompressed )
+                                file_err( 'Problem decompressing zipped data', dataset, json_file )
+                                return
+                    z.close()
+                    # Replace the zipped file with the decompressed file if it's safe to do so
+                    if uncompressed is not None:
+                        if dataset.type in ( 'server_dir', 'path_paste' ) or not in_place:
+                            dataset.path = uncompressed
+                        else:
+                            shutil.move( uncompressed, dataset.path )
+                        os.chmod(dataset.path, 0644)
+                        dataset.name = uncompressed_name
+                data_type = 'zip'
+        if not data_type:
+            if check_binary( dataset.path ):
+                # We have a binary dataset, but it is not Bam, Sff or Pdf
+                data_type = 'binary'
+                #binary_ok = False
+                parts = dataset.name.split( "." )
+                if len( parts ) > 1:
+                    ext = parts[1].strip().lower()
+                    if not Binary.is_ext_unsniffable(ext):
+                        file_err( 'The uploaded binary file contains inappropriate content', dataset, json_file )
+                        return
+                    elif Binary.is_ext_unsniffable(ext) and dataset.file_type != ext:
+                        err_msg = "You must manually set the 'File Format' to '%s' when uploading %s files." % ( ext.capitalize(), ext )
+                        file_err( err_msg, dataset, json_file )
+                        return
+        if not data_type:
+            # We must have a text file
+            if check_html( dataset.path ):
+                file_err( 'The uploaded file contains inappropriate HTML content', dataset, json_file )
+                return
+        if data_type != 'binary':
+            if link_data_only == 'copy_files':
+                if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]:
+                    in_place = False
+                if dataset.space_to_tab:
+                    line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place )
+                else:
+                    line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place )
+            if dataset.file_type == 'auto':
+                ext = sniff.guess_ext( dataset.path, registry.sniff_order )
+            else:
+                ext = dataset.file_type
+            data_type = ext
+    # Save job info for the framework
+    if ext == 'auto' and dataset.ext:
+        ext = dataset.ext
+    if ext == 'auto':
+        ext = 'data'
+    datatype = registry.get_datatype_by_extension( ext )
+    if dataset.type in ( 'server_dir', 'path_paste' ) and link_data_only == 'link_to_files':
+        # Never alter a file that will not be copied to Galaxy's local file store.
+        if datatype.dataset_content_needs_grooming( dataset.path ):
+            err_msg = 'The uploaded files need grooming, so change your <b>Copy data into Galaxy?</b> selection to be ' + \
+                '<b>Copy files into Galaxy</b> instead of <b>Link to files without copying into Galaxy</b> so grooming can be performed.'
+            file_err( err_msg, dataset, json_file )
+            return
+    if link_data_only == 'copy_files' and dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]:
+        # Move the dataset to its "real" path
+        if converted_path is not None:
+            shutil.copy( converted_path, output_path )
+            try:
+                os.remove( converted_path )
+            except:
+                pass
+        else:
+            # This should not happen, but it's here just in case
+            shutil.copy( dataset.path, output_path )
+    elif link_data_only == 'copy_files':
+        shutil.move( dataset.path, output_path )
+    # Write the job info
+    stdout = stdout or 'uploaded %s file' % data_type
+    info = dict( type = 'dataset',
+                 dataset_id = dataset.dataset_id,
+                 ext = ext,
+                 stdout = stdout,
+                 name = dataset.name,
+                 line_count = line_count )
+    json_file.write( to_json_string( info ) + "\n" )
 
+    if link_data_only == 'copy_files' and datatype.dataset_content_needs_grooming( output_path ):
+        # Groom the dataset content if necessary
+        datatype.groom_dataset_content( output_path )
 
-  
-  
-<script type="text/javascript" src="https://dwz7u9t8u8usb.cloudfront.net/m/33b98326237f/compressed/js/1e1f2fcf08d4.js"></script>
+def add_composite_file( dataset, registry, json_file, output_path, files_path ):
+        if dataset.composite_files:
+            os.mkdir( files_path )
+            for name, value in dataset.composite_files.iteritems():
+                value = util.bunch.Bunch( **value )
+                if dataset.composite_file_paths[ value.name ] is None and not value.optional:
+                    file_err( 'A required composite data file was not provided (%s)' % name, dataset, json_file )
+                    break
+                elif dataset.composite_file_paths[value.name] is not None:
+                    dp = dataset.composite_file_paths[value.name][ 'path' ]
+                    isurl = dp.find('://') <> -1 # todo fixme
+                    if isurl:
+                       try:
+                           temp_name, dataset.is_multi_byte = sniff.stream_to_file( urllib.urlopen( dp ), prefix='url_paste' )
+                       except Exception, e:
+                           file_err( 'Unable to fetch %s\n%s' % ( dp, str( e ) ), dataset, json_file )
+                           return
+                       dataset.path = temp_name
+                       dp = temp_name
+                    if not value.is_binary:
+                        if dataset.composite_file_paths[ value.name ].get( 'space_to_tab', value.space_to_tab ):
+                            sniff.convert_newlines_sep2tabs( dp )
+                        else:
+                            sniff.convert_newlines( dp )
+                    shutil.move( dp, os.path.join( files_path, name ) )
+        # Move the dataset to its "real" path
+        shutil.move( dataset.primary_file, output_path )
+        # Write the job info
+        info = dict( type = 'dataset',
+                     dataset_id = dataset.dataset_id,
+                     stdout = 'uploaded %s file' % dataset.file_type )
+        json_file.write( to_json_string( info ) + "\n" )
 
+def __main__():
 
+    if len( sys.argv ) < 4:
+        print >>sys.stderr, 'usage: upload.py <root> <datatypes_conf> <json paramfile> <output spec> ...'
+        sys.exit( 1 )
 
+    output_paths = parse_outputs( sys.argv[4:] )
+    json_file = open( 'galaxy.json', 'w' )
 
-</head>
+    registry = Registry()
+    registry.load_datatypes( root_dir=sys.argv[1], config=sys.argv[2] )
 
-<body id="" class=" ">
-  <script>
-    if (navigator.userAgent.indexOf(' AppleWebKit/') === -1) {
-      $('body').addClass('non-webkit')
-    }
-    $('body')
-      .addClass($.client.os.toLowerCase())
-      .addClass($.client.browser.toLowerCase())
-  </script>
-  <!--[if IE 8]>
-  <script>jQuery(document.body).addClass('ie8')</script>
-  <![endif]-->
-  <!--[if IE 9]>
-  <script>jQuery(document.body).addClass('ie9')</script>
-  <![endif]-->
+    for line in open( sys.argv[3], 'r' ):
+        dataset = from_json_string( line )
+        dataset = util.bunch.Bunch( **safe_dict( dataset ) )
+        try:
+            output_path = output_paths[int( dataset.dataset_id )][0]
+        except:
+            print >>sys.stderr, 'Output path for dataset %s not found on command line' % dataset.dataset_id
+            sys.exit( 1 )
+        if dataset.type == 'composite':
+            files_path = output_paths[int( dataset.dataset_id )][1]
+            add_composite_file( dataset, registry, json_file, output_path, files_path )
+        else:
+            add_file( dataset, registry, json_file, output_path )
 
-  <div id="wrapper">
+    # clean up paramfile
+    # TODO: this will not work when running as the actual user unless the
+    # parent directory is writable by the user.
+    try:
+        os.remove( sys.argv[3] )
+    except:
+        pass
 
-
-
-  <div id="header-wrap">
-    <div id="header">
-    <ul id="global-nav">
-      <li><a class="home" href="http://www.atlassian.com">Atlassian Home</a></li>
-      <li><a class="docs" href="http://confluence.atlassian.com/display/BITBUCKET">Documentation</a></li>
-      <li><a class="support" href="/support">Support</a></li>
-      <li><a class="blog" href="http://blog.bitbucket.org">Blog</a></li>
-      <li><a class="forums" href="http://groups.google.com/group/bitbucket-users">Forums</a></li>
-    </ul>
-    <a href="/" id="logo">Bitbucket by Atlassian</a>
-
-    <div id="main-nav">
-    
-      <ul class="clearfix">
-        <li><a href="/plans">Pricing &amp; signup</a></li>
-        <li><a id="explore-link" href="/explore">Explore Bitbucket</a></li>
-        <li><a href="/account/signin/?next=/galaxy/galaxy-central/src/05fc04a70a3b/tools/data_source/upload.py">Log in</a></li>
-        
-
-<li class="search-box">
-  <form action="/repo/all">
-    <input type="search" results="5" autosave="bitbucket-explore-search"
-           name="name" id="searchbox"
-           placeholder="owner/repo" />
-  </form>
-</li>
-
-      </ul>
-    
-    </div>
-
-  
-
-    </div>
-  </div>
-
-    <div id="header-messages">
-  
-    
-    
-  
-
-    
-   </div>
-
-
-
-    <div id="content">
-      <div id="source">
-      
-  
-  
-
-
-
-
-
-  <script>
-    jQuery(function ($) {
-        var cookie = $.cookie,
-            cookieOptions, date,
-            $content = $('#content'),
-            $pane = $('#what-is-bitbucket'),
-            $hide = $pane.find('[href="#hide"]').css('display', 'block').hide();
-
-        date = new Date();
-        date.setTime(date.getTime() + 365 * 24 * 60 * 60 * 1000);
-        cookieOptions = { path: '/', expires: date };
-
-        if (cookie('toggle_status') == 'hide') $content.addClass('repo-desc-hidden');
-
-        $('#toggle-repo-content').click(function (event) {
-            event.preventDefault();
-            $content.toggleClass('repo-desc-hidden');
-            cookie('toggle_status', cookie('toggle_status') == 'show' ? 'hide' : 'show', cookieOptions);
-        });
-
-        if (!cookie('hide_intro_message')) $pane.show();
-
-        $hide.click(function (event) {
-            event.preventDefault();
-            cookie('hide_intro_message', true, cookieOptions);
-            $pane.slideUp('slow');
-        });
-
-        $pane.hover(
-            function () { $hide.fadeIn('fast'); },
-            function () { $hide.fadeOut('fast'); });
-
-      (function () {
-        // Update "recently-viewed-repos" cookie for
-        // the "repositories" drop-down.
-        var
-          id = BB.repo.id,
-          cookieName = 'recently-viewed-repos_' + BB.user.id,
-          rvr = cookie(cookieName),
-          ids = rvr? rvr.split(','): [],
-          idx = _.indexOf(ids, '' + id);
-
-        // Remove `id` from `ids` if present.
-        if (~idx) ids.splice(idx, 1);
-
-        cookie(
-          cookieName,
-          // Insert `id` as the first item, then call
-          // `join` on the resulting array to produce
-          // something like "114694,27542,89002,84570".
-          [id].concat(ids.slice(0, 4)).join(),
-          {path: '/', expires: 1e6} // "never" expires
-        );
-      }());
-    });
-  </script>
-
-
-
-    <div id="what-is-bitbucket" class="new-to-bitbucket">
-      <h2>galaxy <span id="slogan">is sharing code with you</span></h2>
-      <img src="https://dwz7u9t8u8usb.cloudfront.net/m/33b98326237f/img/default_avatar/32/user_blue.png" alt="" class="avatar" />
-      <p>Bitbucket is a code hosting site. Unlimited public and private repositories. Free for small teams.</p>
-      <div class="primary-action-link signup"><a href="/account/signup/?utm_source=internal&utm_medium=banner&utm_campaign=what_is_bitbucket">Try Bitbucket free</a></div>
-      <a href="#hide" title="Don't show this again">Don't show this again</a>
-    </div>
-
-
-<div id="tabs" class="tabs">
-  <ul>
-    
-      <li>
-        <a href="/galaxy/galaxy-central/overview" id="repo-overview-link">Overview</a>
-      </li>
-    
-
-    
-      <li>
-        <a href="/galaxy/galaxy-central/downloads" id="repo-downloads-link">Downloads (<span id="downloads-count">0</span>)</a>
-      </li>
-    
-
-    
-      
-    
-
-    
-      <li>
-        <a href="/galaxy/galaxy-central/pull-requests" id="repo-pr-link">Pull requests (13)</a>
-      </li>
-    
-
-    
-      <li class="selected">
-        
-          <a href="/galaxy/galaxy-central/src" id="repo-source-link">Source</a>
-        
-      </li>
-    
-
-    
-      <li>
-        <a href="/galaxy/galaxy-central/changesets" id="repo-commits-link">Commits</a>
-      </li>
-    
-
-    <li id="wiki-tab" class="dropdown"
-      style="display:
-          block 
-        
-      ">
-      <a href="/galaxy/galaxy-central/wiki" id="repo-wiki-link">Wiki</a>
-    </li>
-
-    <li id="issues-tab" class="dropdown inertial-hover"
-      style="display:
-                                              none  
-        
-      ">
-      <a href="/galaxy/galaxy-central/issues?status=new&amp;status=open" id="repo-issues-link">Issues (263) &raquo;</a>
-      <ul>
-        <li><a href="/galaxy/galaxy-central/issues/new">Create issue</a></li>
-        <li><a href="/galaxy/galaxy-central/issues?status=new">New issues</a></li>
-        <li><a href="/galaxy/galaxy-central/issues?status=new&amp;status=open">Open issues</a></li>
-        <li><a href="/galaxy/galaxy-central/issues?status=duplicate&amp;status=invalid&amp;status=resolved&amp;status=wontfix">Closed issues</a></li>
-        
-        <li><a href="/galaxy/galaxy-central/issues">All issues</a></li>
-        <li><a href="/galaxy/galaxy-central/issues/query">Advanced query</a></li>
-      </ul>
-    </li>
-
-    
-  </ul>
-
-  <ul>
-    
-      <li>
-        <a href="/galaxy/galaxy-central/descendants" id="repo-forks-link">Forks/queues (108)</a>
-      </li>
-    
-
-    
-      <li>
-        <a href="/galaxy/galaxy-central/zealots">Followers (<span id="followers-count">147</span>)</a>
-      </li>
-    
-  </ul>
-</div>
-
- 
-
-
-  <div class="repo-menu" id="repo-menu">
-    <ul id="repo-menu-links">
-    
-    
-      <li>
-        <a href="/galaxy/galaxy-central/rss" class="rss" title="RSS feed for galaxy-central">RSS</a>
-      </li>
-
-      <li><a id="repo-fork-link" href="/galaxy/galaxy-central/fork" class="fork">fork</a></li>
-      
-        
-          <li><a id="repo-patch-queue-link" href="/galaxy/galaxy-central/hack" class="patch-queue">patch queue</a></li>
-        
-      
-      <li>
-        <a id="repo-follow-link" rel="nofollow" href="/galaxy/galaxy-central/follow" class="follow">follow</a>
-      </li>
-      
-          
-      
-      
-        <li class="get-source inertial-hover">
-          <a class="source">get source</a>
-          <ul class="downloads">
-            
-              
-              <li><a rel="nofollow" href="/galaxy/galaxy-central/get/05fc04a70a3b.zip">zip</a></li>
-              <li><a rel="nofollow" href="/galaxy/galaxy-central/get/05fc04a70a3b.tar.gz">gz</a></li>
-              <li><a rel="nofollow" href="/galaxy/galaxy-central/get/05fc04a70a3b.tar.bz2">bz2</a></li>
-            
-          </ul>
-        </li>
-      
-      
-    </ul>
-
-  
-    <ul class="metadata">
-      
-      
-      
-        <li class="branches inertial-hover">branches
-          <ul>
-            <li class="filter">
-              <input type="text" class="dropdown-filter" placeholder="Filter branches" autosave="branch-dropdown-6304"/>
-            </li>
-            
-            <li class="comprev"><a href="/galaxy/galaxy-central/src/05fc04a70a3b" title="default">default</a>
-              
-            </li>
-          </ul>
-        </li>
-      
-      
-      <li class="tags inertial-hover">tags
-        <ul>
-          <li class="filter">
-            <input type="text" class="dropdown-filter" placeholder="Filter tags" autosave="tags-dropdown-6304"/>
-          </li>
-          <li class="comprev"><a href="/galaxy/galaxy-central/src/05fc04a70a3b">tip</a>
-            </li>
-        </ul>
-      </li>
-     
-     
-      
-    </ul>
-  
-  </div>
-
-
-
-
-<div class="repo-menu" id="repo-desc">
-    <ul id="repo-menu-links-mini">
-      <li><a rel="nofollow" class="compare-link"
-             href="/galaxy/galaxy-central/compare"
-             title="Show changes between galaxy-central and "
-             ></a></li>
-      
-
-      
-      <li>
-        <a href="/galaxy/galaxy-central/rss" class="rss" title="RSS feed for galaxy-central"></a>
-      </li>
-
-      <li><a href="/galaxy/galaxy-central/fork" class="fork" title="Fork"></a></li>
-      
-        
-          <li><a href="/galaxy/galaxy-central/hack" class="patch-queue" title="Patch queue"></a></li>
-        
-      
-      <li>
-        <a rel="nofollow" href="/galaxy/galaxy-central/follow" class="follow">follow</a>
-      </li>
-      
-          
-      
-      
-        <li>
-          <a class="source" title="Get source"></a>
-          <ul class="downloads">
-            
-              
-              <li><a rel="nofollow" href="/galaxy/galaxy-central/get/05fc04a70a3b.zip">zip</a></li>
-              <li><a rel="nofollow" href="/galaxy/galaxy-central/get/05fc04a70a3b.tar.gz">gz</a></li>
-              <li><a rel="nofollow" href="/galaxy/galaxy-central/get/05fc04a70a3b.tar.bz2">bz2</a></li>
-            
-          </ul>
-        </li>
-      
-    </ul>
-
-    <h3 id="repo-heading" class="public hg">
-      <a class="owner-username" href="/galaxy">galaxy</a> /
-      <a class="repo-name" href="/galaxy/galaxy-central">galaxy-central</a>
-    
-      <span><a href="http://galaxyproject.org/">http://galaxyproject.org/</a></span>
-    
-
-    <ul id="fork-actions" class="button-group">
-      <li>
-        <a id="repo-compare-link"
-           href="/galaxy/galaxy-central/compare"
-           rel="nofollow" class="icon compare-link">compare</a>
-      </li>
-    
-    </ul>
-    </h3>
-
-    
-      <p class="repo-desc-description">Main development repository for Galaxy. 
-Active development happens here, and this repository is thus intended for those working on Galaxy development. See <a href="http://bitbucket.org/galaxy/galaxy-dist/" rel="nofollow">http://bitbucket.org/galaxy/galaxy-dist/</a> for a more stable repository intended for end-users.  The project homepage is <a href="http://galaxyproject.org" rel="nofollow">http://galaxyproject.org</a> and the wiki is <a href="http://galaxyproject.org/wiki" rel="nofollow">http://galaxyproject.org/wiki</a></p>
-    
-
-  <div id="repo-desc-cloneinfo">Clone this repository (size: 222.4 MB):
-    <a href="https://bitbucket.org/galaxy/galaxy-central" class="https">HTTPS</a> /
-    <a href="ssh://hg@bitbucket.org/galaxy/galaxy-central" class="ssh">SSH</a>
-    <div id="sourcetree-clone-link" class="clone-in-client mac anonymous help-activated"
-         data-desktop-clone-url-ssh="ssh://hg@bitbucket.org/galaxy/galaxy-central"
-         data-desktop-clone-url-https="https://bitbucket.org/galaxy/galaxy-central">
-        /
-      <a class="desktop-ssh"
-         href="sourcetree://cloneRepo/ssh://hg@bitbucket.org/galaxy/galaxy-central">SourceTree</a>
-      <a class="desktop-https"
-         href="sourcetree://cloneRepo/https://bitbucket.org/galaxy/galaxy-central">SourceTree</a>
-    </div>
-    
-    <pre id="clone-url-https">hg clone https://bitbucket.org/galaxy/galaxy-central</pre>
-    <pre id="clone-url-ssh">hg clone ssh://hg@bitbucket.org/galaxy/galaxy-central</pre>
-    
-  </div>
-
-        <a href="#" id="toggle-repo-content"></a>
-
-        
-
-        
-          
-        
-
-</div>
-
-
-
-
-      
-  <div id="source-container">
-    
-
-  <div id="source-path">
-    <h1>
-      <a href="/galaxy/galaxy-central/src" class="src-pjax">galaxy-central</a> /
-
-  
-    
-      <a href="/galaxy/galaxy-central/src/05fc04a70a3b/tools/" class="src-pjax">tools</a> /
-    
-  
-
-  
-    
-      <a href="/galaxy/galaxy-central/src/05fc04a70a3b/tools/data_source/" class="src-pjax">data_source</a> /
-    
-  
-
-  
-    
-      <span>upload.py</span>
-    
-  
-
-    </h1>
-  </div>
-
-  <div class="labels labels-csv">
-  
-    <dl>
-  
-  
-  
-    <dt>Branch</dt>
-    
-      
-        <dd class="branch unabridged"><a href="/galaxy/galaxy-central/changesets/tip/branch(%22default%22)" title="default">default</a></dd>
-      
-    
-  
-</dl>
-
-  
-  </div>
-
-
-  
-  <div id="source-view">
-    <div class="header">
-      <ul class="metadata">
-        <li><code>05fc04a70a3b</code></li>
-        
-          
-            <li>384 loc</li>
-          
-        
-        <li>17.5 KB</li>
-      </ul>
-      <ul class="source-view-links">
-        
-        <li><a id="embed-link" href="https://bitbucket.org/galaxy/galaxy-central/src/05fc04a70a3b/tools/data_source/upload.py?embed=t">embed</a></li>
-        
-        <li><a href="/galaxy/galaxy-central/history/tools/data_source/upload.py">history</a></li>
-        
-        <li><a href="/galaxy/galaxy-central/annotate/05fc04a70a3b/tools/data_source/upload.py">annotate</a></li>
-        
-        <li><a href="/galaxy/galaxy-central/raw/05fc04a70a3b/tools/data_source/upload.py">raw</a></li>
-        <li>
-          <form action="/galaxy/galaxy-central/diff/tools/data_source/upload.py" class="source-view-form">
-          
-            <input type="hidden" name="diff2" value="3338d7629e33" />
-            <select name="diff1">
-            
-              
-                <option value="3338d7629e33">3338d7629e33</option>
-              
-            
-              
-                <option value="8bccff9db869">8bccff9db869</option>
-              
-            
-              
-                <option value="577498958c37">577498958c37</option>
-              
-            
-              
-                <option value="7c3c50784374">7c3c50784374</option>
-              
-            
-              
-                <option value="419260098a6f">419260098a6f</option>
-              
-            
-              
-                <option value="2aa927225355">2aa927225355</option>
-              
-            
-              
-                <option value="1d0bb560bd3d">1d0bb560bd3d</option>
-              
-            
-              
-                <option value="ab90893a7cf5">ab90893a7cf5</option>
-              
-            
-              
-                <option value="4b3fefa396cf">4b3fefa396cf</option>
-              
-            
-              
-                <option value="e7214c69ed7d">e7214c69ed7d</option>
-              
-            
-              
-                <option value="2ad3ef1aecc6">2ad3ef1aecc6</option>
-              
-            
-              
-                <option value="c2bc73b017cb">c2bc73b017cb</option>
-              
-            
-              
-                <option value="b5ecb8f4839d">b5ecb8f4839d</option>
-              
-            
-              
-                <option value="70d7d0bfaad6">70d7d0bfaad6</option>
-              
-            
-              
-                <option value="341139a7e3ea">341139a7e3ea</option>
-              
-            
-              
-                <option value="3c32bd44d95b">3c32bd44d95b</option>
-              
-            
-              
-                <option value="cae28ae0ac46">cae28ae0ac46</option>
-              
-            
-              
-                <option value="5ee2200b2b6d">5ee2200b2b6d</option>
-              
-            
-              
-                <option value="5f9d6d9582e5">5f9d6d9582e5</option>
-              
-            
-              
-                <option value="d3c338e3e08a">d3c338e3e08a</option>
-              
-            
-              
-                <option value="d71a58ccc49f">d71a58ccc49f</option>
-              
-            
-              
-                <option value="03be4a0828ce">03be4a0828ce</option>
-              
-            
-              
-                <option value="67f5a7f16c59">67f5a7f16c59</option>
-              
-            
-              
-                <option value="150c8db8dec1">150c8db8dec1</option>
-              
-            
-              
-                <option value="546e644ae6f9">546e644ae6f9</option>
-              
-            
-              
-                <option value="37f4420ba3fc">37f4420ba3fc</option>
-              
-            
-              
-                <option value="b762ea8e5508">b762ea8e5508</option>
-              
-            
-              
-                <option value="fabf48e747ae">fabf48e747ae</option>
-              
-            
-              
-                <option value="8f6fa47b21c4">8f6fa47b21c4</option>
-              
-            
-              
-                <option value="0c9e154e9176">0c9e154e9176</option>
-              
-            
-              
-                <option value="0ba4a2b77f65">0ba4a2b77f65</option>
-              
-            
-              
-                <option value="022a8c94883f">022a8c94883f</option>
-              
-            
-              
-                <option value="8feff3bc14bc">8feff3bc14bc</option>
-              
-            
-              
-                <option value="495dda4c0693">495dda4c0693</option>
-              
-            
-              
-                <option value="23c1b9799bce">23c1b9799bce</option>
-              
-            
-              
-                <option value="18586d1194f9">18586d1194f9</option>
-              
-            
-              
-                <option value="1ce07e82a4c1">1ce07e82a4c1</option>
-              
-            
-              
-                <option value="15756ebb2b11">15756ebb2b11</option>
-              
-            
-              
-                <option value="d31ab50dc8e0">d31ab50dc8e0</option>
-              
-            
-              
-                <option value="88fdb7f40b0e">88fdb7f40b0e</option>
-              
-            
-              
-                <option value="3559f7377b9c">3559f7377b9c</option>
-              
-            
-              
-                <option value="d3fe789e3931">d3fe789e3931</option>
-              
-            
-              
-                <option value="16b1ca0b2746">16b1ca0b2746</option>
-              
-            
-              
-                <option value="d0fb5c640d43">d0fb5c640d43</option>
-              
-            
-              
-                <option value="f6e0863862ef">f6e0863862ef</option>
-              
-            
-            </select>
-            <input type="submit" value="diff" />
-          
-          </form>
-        </li>
-        
-      </ul>
-    </div>
-  
-    
-      
-        <div>
-          <table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><a href="#cl-1">  1</a>
-<a href="#cl-2">  2</a>
-<a href="#cl-3">  3</a>
-<a href="#cl-4">  4</a>
-<a href="#cl-5">  5</a>
-<a href="#cl-6">  6</a>
-<a href="#cl-7">  7</a>
-<a href="#cl-8">  8</a>
-<a href="#cl-9">  9</a>
-<a href="#cl-10"> 10</a>
-<a href="#cl-11"> 11</a>
-<a href="#cl-12"> 12</a>
-<a href="#cl-13"> 13</a>
-<a href="#cl-14"> 14</a>
-<a href="#cl-15"> 15</a>
-<a href="#cl-16"> 16</a>
-<a href="#cl-17"> 17</a>
-<a href="#cl-18"> 18</a>
-<a href="#cl-19"> 19</a>
-<a href="#cl-20"> 20</a>
-<a href="#cl-21"> 21</a>
-<a href="#cl-22"> 22</a>
-<a href="#cl-23"> 23</a>
-<a href="#cl-24"> 24</a>
-<a href="#cl-25"> 25</a>
-<a href="#cl-26"> 26</a>
-<a href="#cl-27"> 27</a>
-<a href="#cl-28"> 28</a>
-<a href="#cl-29"> 29</a>
-<a href="#cl-30"> 30</a>
-<a href="#cl-31"> 31</a>
-<a href="#cl-32"> 32</a>
-<a href="#cl-33"> 33</a>
-<a href="#cl-34"> 34</a>
-<a href="#cl-35"> 35</a>
-<a href="#cl-36"> 36</a>
-<a href="#cl-37"> 37</a>
-<a href="#cl-38"> 38</a>
-<a href="#cl-39"> 39</a>
-<a href="#cl-40"> 40</a>
-<a href="#cl-41"> 41</a>
-<a href="#cl-42"> 42</a>
-<a href="#cl-43"> 43</a>
-<a href="#cl-44"> 44</a>
-<a href="#cl-45"> 45</a>
-<a href="#cl-46"> 46</a>
-<a href="#cl-47"> 47</a>
-<a href="#cl-48"> 48</a>
-<a href="#cl-49"> 49</a>
-<a href="#cl-50"> 50</a>
-<a href="#cl-51"> 51</a>
-<a href="#cl-52"> 52</a>
-<a href="#cl-53"> 53</a>
-<a href="#cl-54"> 54</a>
-<a href="#cl-55"> 55</a>
-<a href="#cl-56"> 56</a>
-<a href="#cl-57"> 57</a>
-<a href="#cl-58"> 58</a>
-<a href="#cl-59"> 59</a>
-<a href="#cl-60"> 60</a>
-<a href="#cl-61"> 61</a>
-<a href="#cl-62"> 62</a>
-<a href="#cl-63"> 63</a>
-<a href="#cl-64"> 64</a>
-<a href="#cl-65"> 65</a>
-<a href="#cl-66"> 66</a>
-<a href="#cl-67"> 67</a>
-<a href="#cl-68"> 68</a>
-<a href="#cl-69"> 69</a>
-<a href="#cl-70"> 70</a>
-<a href="#cl-71"> 71</a>
-<a href="#cl-72"> 72</a>
-<a href="#cl-73"> 73</a>
-<a href="#cl-74"> 74</a>
-<a href="#cl-75"> 75</a>
-<a href="#cl-76"> 76</a>
-<a href="#cl-77"> 77</a>
-<a href="#cl-78"> 78</a>
-<a href="#cl-79"> 79</a>
-<a href="#cl-80"> 80</a>
-<a href="#cl-81"> 81</a>
-<a href="#cl-82"> 82</a>
-<a href="#cl-83"> 83</a>
-<a href="#cl-84"> 84</a>
-<a href="#cl-85"> 85</a>
-<a href="#cl-86"> 86</a>
-<a href="#cl-87"> 87</a>
-<a href="#cl-88"> 88</a>
-<a href="#cl-89"> 89</a>
-<a href="#cl-90"> 90</a>
-<a href="#cl-91"> 91</a>
-<a href="#cl-92"> 92</a>
-<a href="#cl-93"> 93</a>
-<a href="#cl-94"> 94</a>
-<a href="#cl-95"> 95</a>
-<a href="#cl-96"> 96</a>
-<a href="#cl-97"> 97</a>
-<a href="#cl-98"> 98</a>
-<a href="#cl-99"> 99</a>
-<a href="#cl-100">100</a>
-<a href="#cl-101">101</a>
-<a href="#cl-102">102</a>
-<a href="#cl-103">103</a>
-<a href="#cl-104">104</a>
-<a href="#cl-105">105</a>
-<a href="#cl-106">106</a>
-<a href="#cl-107">107</a>
-<a href="#cl-108">108</a>
-<a href="#cl-109">109</a>
-<a href="#cl-110">110</a>
-<a href="#cl-111">111</a>
-<a href="#cl-112">112</a>
-<a href="#cl-113">113</a>
-<a href="#cl-114">114</a>
-<a href="#cl-115">115</a>
-<a href="#cl-116">116</a>
-<a href="#cl-117">117</a>
-<a href="#cl-118">118</a>
-<a href="#cl-119">119</a>
-<a href="#cl-120">120</a>
-<a href="#cl-121">121</a>
-<a href="#cl-122">122</a>
-<a href="#cl-123">123</a>
-<a href="#cl-124">124</a>
-<a href="#cl-125">125</a>
-<a href="#cl-126">126</a>
-<a href="#cl-127">127</a>
-<a href="#cl-128">128</a>
-<a href="#cl-129">129</a>
-<a href="#cl-130">130</a>
-<a href="#cl-131">131</a>
-<a href="#cl-132">132</a>
-<a href="#cl-133">133</a>
-<a href="#cl-134">134</a>
-<a href="#cl-135">135</a>
-<a href="#cl-136">136</a>
-<a href="#cl-137">137</a>
-<a href="#cl-138">138</a>
-<a href="#cl-139">139</a>
-<a href="#cl-140">140</a>
-<a href="#cl-141">141</a>
-<a href="#cl-142">142</a>
-<a href="#cl-143">143</a>
-<a href="#cl-144">144</a>
-<a href="#cl-145">145</a>
-<a href="#cl-146">146</a>
-<a href="#cl-147">147</a>
-<a href="#cl-148">148</a>
-<a href="#cl-149">149</a>
-<a href="#cl-150">150</a>
-<a href="#cl-151">151</a>
-<a href="#cl-152">152</a>
-<a href="#cl-153">153</a>
-<a href="#cl-154">154</a>
-<a href="#cl-155">155</a>
-<a href="#cl-156">156</a>
-<a href="#cl-157">157</a>
-<a href="#cl-158">158</a>
-<a href="#cl-159">159</a>
-<a href="#cl-160">160</a>
-<a href="#cl-161">161</a>
-<a href="#cl-162">162</a>
-<a href="#cl-163">163</a>
-<a href="#cl-164">164</a>
-<a href="#cl-165">165</a>
-<a href="#cl-166">166</a>
-<a href="#cl-167">167</a>
-<a href="#cl-168">168</a>
-<a href="#cl-169">169</a>
-<a href="#cl-170">170</a>
-<a href="#cl-171">171</a>
-<a href="#cl-172">172</a>
-<a href="#cl-173">173</a>
-<a href="#cl-174">174</a>
-<a href="#cl-175">175</a>
-<a href="#cl-176">176</a>
-<a href="#cl-177">177</a>
-<a href="#cl-178">178</a>
-<a href="#cl-179">179</a>
-<a href="#cl-180">180</a>
-<a href="#cl-181">181</a>
-<a href="#cl-182">182</a>
-<a href="#cl-183">183</a>
-<a href="#cl-184">184</a>
-<a href="#cl-185">185</a>
-<a href="#cl-186">186</a>
-<a href="#cl-187">187</a>
-<a href="#cl-188">188</a>
-<a href="#cl-189">189</a>
-<a href="#cl-190">190</a>
-<a href="#cl-191">191</a>
-<a href="#cl-192">192</a>
-<a href="#cl-193">193</a>
-<a href="#cl-194">194</a>
-<a href="#cl-195">195</a>
-<a href="#cl-196">196</a>
-<a href="#cl-197">197</a>
-<a href="#cl-198">198</a>
-<a href="#cl-199">199</a>
-<a href="#cl-200">200</a>
-<a href="#cl-201">201</a>
-<a href="#cl-202">202</a>
-<a href="#cl-203">203</a>
-<a href="#cl-204">204</a>
-<a href="#cl-205">205</a>
-<a href="#cl-206">206</a>
-<a href="#cl-207">207</a>
-<a href="#cl-208">208</a>
-<a href="#cl-209">209</a>
-<a href="#cl-210">210</a>
-<a href="#cl-211">211</a>
-<a href="#cl-212">212</a>
-<a href="#cl-213">213</a>
-<a href="#cl-214">214</a>
-<a href="#cl-215">215</a>
-<a href="#cl-216">216</a>
-<a href="#cl-217">217</a>
-<a href="#cl-218">218</a>
-<a href="#cl-219">219</a>
-<a href="#cl-220">220</a>
-<a href="#cl-221">221</a>
-<a href="#cl-222">222</a>
-<a href="#cl-223">223</a>
-<a href="#cl-224">224</a>
-<a href="#cl-225">225</a>
-<a href="#cl-226">226</a>
-<a href="#cl-227">227</a>
-<a href="#cl-228">228</a>
-<a href="#cl-229">229</a>
-<a href="#cl-230">230</a>
-<a href="#cl-231">231</a>
-<a href="#cl-232">232</a>
-<a href="#cl-233">233</a>
-<a href="#cl-234">234</a>
-<a href="#cl-235">235</a>
-<a href="#cl-236">236</a>
-<a href="#cl-237">237</a>
-<a href="#cl-238">238</a>
-<a href="#cl-239">239</a>
-<a href="#cl-240">240</a>
-<a href="#cl-241">241</a>
-<a href="#cl-242">242</a>
-<a href="#cl-243">243</a>
-<a href="#cl-244">244</a>
-<a href="#cl-245">245</a>
-<a href="#cl-246">246</a>
-<a href="#cl-247">247</a>
-<a href="#cl-248">248</a>
-<a href="#cl-249">249</a>
-<a href="#cl-250">250</a>
-<a href="#cl-251">251</a>
-<a href="#cl-252">252</a>
-<a href="#cl-253">253</a>
-<a href="#cl-254">254</a>
-<a href="#cl-255">255</a>
-<a href="#cl-256">256</a>
-<a href="#cl-257">257</a>
-<a href="#cl-258">258</a>
-<a href="#cl-259">259</a>
-<a href="#cl-260">260</a>
-<a href="#cl-261">261</a>
-<a href="#cl-262">262</a>
-<a href="#cl-263">263</a>
-<a href="#cl-264">264</a>
-<a href="#cl-265">265</a>
-<a href="#cl-266">266</a>
-<a href="#cl-267">267</a>
-<a href="#cl-268">268</a>
-<a href="#cl-269">269</a>
-<a href="#cl-270">270</a>
-<a href="#cl-271">271</a>
-<a href="#cl-272">272</a>
-<a href="#cl-273">273</a>
-<a href="#cl-274">274</a>
-<a href="#cl-275">275</a>
-<a href="#cl-276">276</a>
-<a href="#cl-277">277</a>
-<a href="#cl-278">278</a>
-<a href="#cl-279">279</a>
-<a href="#cl-280">280</a>
-<a href="#cl-281">281</a>
-<a href="#cl-282">282</a>
-<a href="#cl-283">283</a>
-<a href="#cl-284">284</a>
-<a href="#cl-285">285</a>
-<a href="#cl-286">286</a>
-<a href="#cl-287">287</a>
-<a href="#cl-288">288</a>
-<a href="#cl-289">289</a>
-<a href="#cl-290">290</a>
-<a href="#cl-291">291</a>
-<a href="#cl-292">292</a>
-<a href="#cl-293">293</a>
-<a href="#cl-294">294</a>
-<a href="#cl-295">295</a>
-<a href="#cl-296">296</a>
-<a href="#cl-297">297</a>
-<a href="#cl-298">298</a>
-<a href="#cl-299">299</a>
-<a href="#cl-300">300</a>
-<a href="#cl-301">301</a>
-<a href="#cl-302">302</a>
-<a href="#cl-303">303</a>
-<a href="#cl-304">304</a>
-<a href="#cl-305">305</a>
-<a href="#cl-306">306</a>
-<a href="#cl-307">307</a>
-<a href="#cl-308">308</a>
-<a href="#cl-309">309</a>
-<a href="#cl-310">310</a>
-<a href="#cl-311">311</a>
-<a href="#cl-312">312</a>
-<a href="#cl-313">313</a>
-<a href="#cl-314">314</a>
-<a href="#cl-315">315</a>
-<a href="#cl-316">316</a>
-<a href="#cl-317">317</a>
-<a href="#cl-318">318</a>
-<a href="#cl-319">319</a>
-<a href="#cl-320">320</a>
-<a href="#cl-321">321</a>
-<a href="#cl-322">322</a>
-<a href="#cl-323">323</a>
-<a href="#cl-324">324</a>
-<a href="#cl-325">325</a>
-<a href="#cl-326">326</a>
-<a href="#cl-327">327</a>
-<a href="#cl-328">328</a>
-<a href="#cl-329">329</a>
-<a href="#cl-330">330</a>
-<a href="#cl-331">331</a>
-<a href="#cl-332">332</a>
-<a href="#cl-333">333</a>
-<a href="#cl-334">334</a>
-<a href="#cl-335">335</a>
-<a href="#cl-336">336</a>
-<a href="#cl-337">337</a>
-<a href="#cl-338">338</a>
-<a href="#cl-339">339</a>
-<a href="#cl-340">340</a>
-<a href="#cl-341">341</a>
-<a href="#cl-342">342</a>
-<a href="#cl-343">343</a>
-<a href="#cl-344">344</a>
-<a href="#cl-345">345</a>
-<a href="#cl-346">346</a>
-<a href="#cl-347">347</a>
-<a href="#cl-348">348</a>
-<a href="#cl-349">349</a>
-<a href="#cl-350">350</a>
-<a href="#cl-351">351</a>
-<a href="#cl-352">352</a>
-<a href="#cl-353">353</a>
-<a href="#cl-354">354</a>
-<a href="#cl-355">355</a>
-<a href="#cl-356">356</a>
-<a href="#cl-357">357</a>
-<a href="#cl-358">358</a>
-<a href="#cl-359">359</a>
-<a href="#cl-360">360</a>
-<a href="#cl-361">361</a>
-<a href="#cl-362">362</a>
-<a href="#cl-363">363</a>
-<a href="#cl-364">364</a>
-<a href="#cl-365">365</a>
-<a href="#cl-366">366</a>
-<a href="#cl-367">367</a>
-<a href="#cl-368">368</a>
-<a href="#cl-369">369</a>
-<a href="#cl-370">370</a>
-<a href="#cl-371">371</a>
-<a href="#cl-372">372</a>
-<a href="#cl-373">373</a>
-<a href="#cl-374">374</a>
-<a href="#cl-375">375</a>
-<a href="#cl-376">376</a>
-<a href="#cl-377">377</a>
-<a href="#cl-378">378</a>
-<a href="#cl-379">379</a>
-<a href="#cl-380">380</a>
-<a href="#cl-381">381</a>
-<a href="#cl-382">382</a>
-<a href="#cl-383">383</a>
-<a href="#cl-384">384</a></pre></div></td><td class="code"><div class="highlight"><pre><a name="cl-1"></a><span class="c">#!/usr/bin/env python</span>
-<a name="cl-2"></a><span class="c">#Processes uploads from the user.</span>
-<a name="cl-3"></a>
-<a name="cl-4"></a><span class="c"># WARNING: Changes in this tool (particularly as related to parsing) may need</span>
-<a name="cl-5"></a><span class="c"># to be reflected in galaxy.web.controllers.tool_runner and galaxy.tools</span>
-<a name="cl-6"></a>
-<a name="cl-7"></a><span class="kn">import</span> <span class="nn">urllib</span><span class="o">,</span> <span class="nn">sys</span><span class="o">,</span> <span class="nn">os</span><span class="o">,</span> <span class="nn">gzip</span><span class="o">,</span> <span class="nn">tempfile</span><span class="o">,</span> <span class="nn">shutil</span><span class="o">,</span> <span class="nn">re</span><span class="o">,</span> <span class="nn">gzip</span><span class="o">,</span> <span class="nn">zipfile</span><span class="o">,</span> <span class="nn">codecs</span><span class="o">,</span> <span class="nn">binascii</span>
-<a name="cl-8"></a><span class="kn">from</span> <span class="nn">galaxy</span> <span class="kn">import</span> <span class="n">eggs</span>
-<a name="cl-9"></a><span class="c"># need to import model before sniff to resolve a circular import dependency</span>
-<a name="cl-10"></a><span class="kn">import</span> <span class="nn">galaxy.model</span>
-<a name="cl-11"></a><span class="kn">from</span> <span class="nn">galaxy.datatypes.checkers</span> <span class="kn">import</span> <span class="o">*</span>
-<a name="cl-12"></a><span class="kn">from</span> <span class="nn">galaxy.datatypes</span> <span class="kn">import</span> <span class="n">sniff</span>
-<a name="cl-13"></a><span class="kn">from</span> <span class="nn">galaxy.datatypes.binary</span> <span class="kn">import</span> <span class="o">*</span>
-<a name="cl-14"></a><span class="kn">from</span> <span class="nn">galaxy.datatypes.images</span> <span class="kn">import</span> <span class="n">Pdf</span>
-<a name="cl-15"></a><span class="kn">from</span> <span class="nn">galaxy.datatypes.registry</span> <span class="kn">import</span> <span class="n">Registry</span>
-<a name="cl-16"></a><span class="kn">from</span> <span class="nn">galaxy</span> <span class="kn">import</span> <span class="n">util</span>
-<a name="cl-17"></a><span class="kn">from</span> <span class="nn">galaxy.datatypes.util.image_util</span> <span class="kn">import</span> <span class="o">*</span>
-<a name="cl-18"></a><span class="kn">from</span> <span class="nn">galaxy.util.json</span> <span class="kn">import</span> <span class="o">*</span>
-<a name="cl-19"></a>
-<a name="cl-20"></a><span class="k">try</span><span class="p">:</span>
-<a name="cl-21"></a>    <span class="kn">import</span> <span class="nn">Image</span> <span class="kn">as</span> <span class="nn">PIL</span>
-<a name="cl-22"></a><span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
-<a name="cl-23"></a>    <span class="k">try</span><span class="p">:</span>
-<a name="cl-24"></a>        <span class="kn">from</span> <span class="nn">PIL</span> <span class="kn">import</span> <span class="n">Image</span> <span class="k">as</span> <span class="n">PIL</span>
-<a name="cl-25"></a>    <span class="k">except</span><span class="p">:</span>
-<a name="cl-26"></a>        <span class="n">PIL</span> <span class="o">=</span> <span class="bp">None</span>
-<a name="cl-27"></a>
-<a name="cl-28"></a><span class="k">try</span><span class="p">:</span>
-<a name="cl-29"></a>    <span class="kn">import</span> <span class="nn">bz2</span>
-<a name="cl-30"></a><span class="k">except</span><span class="p">:</span>
-<a name="cl-31"></a>    <span class="n">bz2</span> <span class="o">=</span> <span class="bp">None</span>
-<a name="cl-32"></a>
-<a name="cl-33"></a><span class="k">assert</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span> <span class="o">&gt;=</span> <span class="p">(</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span> <span class="p">)</span>
-<a name="cl-34"></a>
-<a name="cl-35"></a><span class="k">def</span> <span class="nf">stop_err</span><span class="p">(</span> <span class="n">msg</span><span class="p">,</span> <span class="n">ret</span><span class="o">=</span><span class="mi">1</span> <span class="p">):</span>
-<a name="cl-36"></a>    <span class="n">sys</span><span class="o">.</span><span class="n">stderr</span><span class="o">.</span><span class="n">write</span><span class="p">(</span> <span class="n">msg</span> <span class="p">)</span>
-<a name="cl-37"></a>    <span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span> <span class="n">ret</span> <span class="p">)</span>
-<a name="cl-38"></a><span class="k">def</span> <span class="nf">file_err</span><span class="p">(</span> <span class="n">msg</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">):</span>
-<a name="cl-39"></a>    <span class="n">json_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span> <span class="n">to_json_string</span><span class="p">(</span> <span class="nb">dict</span><span class="p">(</span> <span class="nb">type</span> <span class="o">=</span> <span class="s">&#39;dataset&#39;</span><span class="p">,</span>
-<a name="cl-40"></a>                                           <span class="n">ext</span> <span class="o">=</span> <span class="s">&#39;data&#39;</span><span class="p">,</span>
-<a name="cl-41"></a>                                           <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span>
-<a name="cl-42"></a>                                           <span class="n">stderr</span> <span class="o">=</span> <span class="n">msg</span> <span class="p">)</span> <span class="p">)</span> <span class="o">+</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span> <span class="p">)</span>
-<a name="cl-43"></a>    <span class="c"># never remove a server-side upload</span>
-<a name="cl-44"></a>    <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span> <span class="s">&#39;server_dir&#39;</span><span class="p">,</span> <span class="s">&#39;path_paste&#39;</span> <span class="p">):</span>
-<a name="cl-45"></a>        <span class="k">return</span>
-<a name="cl-46"></a>    <span class="k">try</span><span class="p">:</span>
-<a name="cl-47"></a>        <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-48"></a>    <span class="k">except</span><span class="p">:</span>
-<a name="cl-49"></a>        <span class="k">pass</span>
-<a name="cl-50"></a><span class="k">def</span> <span class="nf">safe_dict</span><span class="p">(</span><span class="n">d</span><span class="p">):</span>
-<a name="cl-51"></a>    <span class="sd">&quot;&quot;&quot;</span>
-<a name="cl-52"></a><span class="sd">    Recursively clone json structure with UTF-8 dictionary keys</span>
-<a name="cl-53"></a><span class="sd">    http://mellowmachines.com/blog/2009/06/exploding-dictionary-with-unicode-keys-as-python-arguments/</span>
-<a name="cl-54"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a name="cl-55"></a>    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
-<a name="cl-56"></a>        <span class="k">return</span> <span class="nb">dict</span><span class="p">([(</span><span class="n">k</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s">&#39;utf-8&#39;</span><span class="p">),</span> <span class="n">safe_dict</span><span class="p">(</span><span class="n">v</span><span class="p">))</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span><span class="n">v</span> <span class="ow">in</span> <span class="n">d</span><span class="o">.</span><span class="n">iteritems</span><span class="p">()])</span>
-<a name="cl-57"></a>    <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
-<a name="cl-58"></a>        <span class="k">return</span> <span class="p">[</span><span class="n">safe_dict</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">d</span><span class="p">]</span>
-<a name="cl-59"></a>    <span class="k">else</span><span class="p">:</span>
-<a name="cl-60"></a>        <span class="k">return</span> <span class="n">d</span>
-<a name="cl-61"></a><span class="k">def</span> <span class="nf">parse_outputs</span><span class="p">(</span> <span class="n">args</span> <span class="p">):</span>
-<a name="cl-62"></a>    <span class="n">rval</span> <span class="o">=</span> <span class="p">{}</span>
-<a name="cl-63"></a>    <span class="k">for</span> <span class="n">arg</span> <span class="ow">in</span> <span class="n">args</span><span class="p">:</span>
-<a name="cl-64"></a>        <span class="nb">id</span><span class="p">,</span> <span class="n">files_path</span><span class="p">,</span> <span class="n">path</span> <span class="o">=</span> <span class="n">arg</span><span class="o">.</span><span class="n">split</span><span class="p">(</span> <span class="s">&#39;:&#39;</span><span class="p">,</span> <span class="mi">2</span> <span class="p">)</span>
-<a name="cl-65"></a>        <span class="n">rval</span><span class="p">[</span><span class="nb">int</span><span class="p">(</span> <span class="nb">id</span> <span class="p">)]</span> <span class="o">=</span> <span class="p">(</span> <span class="n">path</span><span class="p">,</span> <span class="n">files_path</span> <span class="p">)</span>
-<a name="cl-66"></a>    <span class="k">return</span> <span class="n">rval</span>
-<a name="cl-67"></a><span class="k">def</span> <span class="nf">add_file</span><span class="p">(</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">registry</span><span class="p">,</span> <span class="n">json_file</span><span class="p">,</span> <span class="n">output_path</span> <span class="p">):</span>
-<a name="cl-68"></a>    <span class="n">data_type</span> <span class="o">=</span> <span class="bp">None</span>
-<a name="cl-69"></a>    <span class="n">line_count</span> <span class="o">=</span> <span class="bp">None</span>
-<a name="cl-70"></a>    <span class="n">converted_path</span> <span class="o">=</span> <span class="bp">None</span>
-<a name="cl-71"></a>    <span class="n">stdout</span> <span class="o">=</span> <span class="bp">None</span>
-<a name="cl-72"></a>    <span class="n">link_data_only</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> <span class="s">&#39;link_data_only&#39;</span><span class="p">,</span> <span class="s">&#39;copy_files&#39;</span> <span class="p">)</span>
-<a name="cl-73"></a>    <span class="n">in_place</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> <span class="s">&#39;in_place&#39;</span><span class="p">,</span> <span class="bp">True</span> <span class="p">)</span>
-<a name="cl-74"></a>
-<a name="cl-75"></a>    <span class="k">try</span><span class="p">:</span>
-<a name="cl-76"></a>        <span class="n">ext</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">file_type</span>
-<a name="cl-77"></a>    <span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
-<a name="cl-78"></a>        <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;Unable to process uploaded file, missing file_type parameter.&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-79"></a>        <span class="k">return</span>
-<a name="cl-80"></a>
-<a name="cl-81"></a>    <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="s">&#39;url&#39;</span><span class="p">:</span>
-<a name="cl-82"></a>        <span class="k">try</span><span class="p">:</span>
-<a name="cl-83"></a>            <span class="n">page</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span> <span class="c">#page will be .close()ed by sniff methods</span>
-<a name="cl-84"></a>            <span class="n">temp_name</span><span class="p">,</span> <span class="n">dataset</span><span class="o">.</span><span class="n">is_multi_byte</span> <span class="o">=</span> <span class="n">sniff</span><span class="o">.</span><span class="n">stream_to_file</span><span class="p">(</span> <span class="n">page</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="s">&#39;url_paste&#39;</span><span class="p">,</span> <span class="n">source_encoding</span><span class="o">=</span><span class="n">util</span><span class="o">.</span><span class="n">get_charset_from_http_headers</span><span class="p">(</span> <span class="n">page</span><span class="o">.</span><span class="n">headers</span> <span class="p">)</span> <span class="p">)</span>
-<a name="cl-85"></a>        <span class="k">except</span> <span class="ne">Exception</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span>
-<a name="cl-86"></a>            <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;Unable to fetch </span><span class="si">%s</span><span class="se">\n</span><span class="si">%s</span><span class="s">&#39;</span> <span class="o">%</span> <span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span> <span class="n">e</span> <span class="p">)</span> <span class="p">),</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-87"></a>            <span class="k">return</span>
-<a name="cl-88"></a>        <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="n">temp_name</span>
-<a name="cl-89"></a>    <span class="c"># See if we have an empty file</span>
-<a name="cl-90"></a>    <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">):</span>
-<a name="cl-91"></a>        <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;Uploaded temporary file (</span><span class="si">%s</span><span class="s">) does not exist.&#39;</span> <span class="o">%</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-92"></a>        <span class="k">return</span>
-<a name="cl-93"></a>    <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getsize</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
-<a name="cl-94"></a>        <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;The uploaded file is empty&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-95"></a>        <span class="k">return</span>
-<a name="cl-96"></a>    <span class="k">if</span> <span class="ow">not</span> <span class="n">dataset</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="s">&#39;url&#39;</span><span class="p">:</span>
-<a name="cl-97"></a>        <span class="c"># Already set is_multi_byte above if type == &#39;url&#39;</span>
-<a name="cl-98"></a>        <span class="k">try</span><span class="p">:</span>
-<a name="cl-99"></a>            <span class="n">dataset</span><span class="o">.</span><span class="n">is_multi_byte</span> <span class="o">=</span> <span class="n">util</span><span class="o">.</span><span class="n">is_multi_byte</span><span class="p">(</span> <span class="n">codecs</span><span class="o">.</span><span class="n">open</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="s">&#39;r&#39;</span><span class="p">,</span> <span class="s">&#39;utf-8&#39;</span> <span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">(</span> <span class="mi">100</span> <span class="p">)</span> <span class="p">)</span>
-<a name="cl-100"></a>        <span class="k">except</span> <span class="ne">UnicodeDecodeError</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span>
-<a name="cl-101"></a>            <span class="n">dataset</span><span class="o">.</span><span class="n">is_multi_byte</span> <span class="o">=</span> <span class="bp">False</span>
-<a name="cl-102"></a>    <span class="c"># Is dataset an image?</span>
-<a name="cl-103"></a>    <span class="n">image</span> <span class="o">=</span> <span class="n">check_image</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-104"></a>    <span class="k">if</span> <span class="n">image</span><span class="p">:</span>
-<a name="cl-105"></a>        <span class="k">if</span> <span class="ow">not</span> <span class="n">PIL</span><span class="p">:</span>
-<a name="cl-106"></a>            <span class="n">image</span> <span class="o">=</span> <span class="bp">None</span>
-<a name="cl-107"></a>        <span class="c"># get_image_ext() returns None if nor a supported Image type</span>
-<a name="cl-108"></a>        <span class="n">ext</span> <span class="o">=</span> <span class="n">get_image_ext</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">image</span> <span class="p">)</span>
-<a name="cl-109"></a>        <span class="n">data_type</span> <span class="o">=</span> <span class="n">ext</span>
-<a name="cl-110"></a>    <span class="c"># Is dataset content multi-byte?</span>
-<a name="cl-111"></a>    <span class="k">elif</span> <span class="n">dataset</span><span class="o">.</span><span class="n">is_multi_byte</span><span class="p">:</span>
-<a name="cl-112"></a>        <span class="n">data_type</span> <span class="o">=</span> <span class="s">&#39;multi-byte char&#39;</span>
-<a name="cl-113"></a>        <span class="n">ext</span> <span class="o">=</span> <span class="n">sniff</span><span class="o">.</span><span class="n">guess_ext</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">is_multi_byte</span><span class="o">=</span><span class="bp">True</span> <span class="p">)</span>
-<a name="cl-114"></a>    <span class="c"># Is dataset content supported sniffable binary?</span>
-<a name="cl-115"></a>    <span class="k">else</span><span class="p">:</span>
-<a name="cl-116"></a>        <span class="n">type_info</span> <span class="o">=</span> <span class="n">Binary</span><span class="o">.</span><span class="n">is_sniffable_binary</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-117"></a>        <span class="k">if</span> <span class="n">type_info</span><span class="p">:</span>
-<a name="cl-118"></a>            <span class="n">data_type</span> <span class="o">=</span> <span class="n">type_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
-<a name="cl-119"></a>            <span class="n">ext</span> <span class="o">=</span> <span class="n">type_info</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
-<a name="cl-120"></a>    <span class="k">if</span> <span class="ow">not</span> <span class="n">data_type</span><span class="p">:</span>
-<a name="cl-121"></a>        <span class="c"># See if we have a gzipped file, which, if it passes our restrictions, we&#39;ll uncompress</span>
-<a name="cl-122"></a>        <span class="n">is_gzipped</span><span class="p">,</span> <span class="n">is_valid</span> <span class="o">=</span> <span class="n">check_gzip</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-123"></a>        <span class="k">if</span> <span class="n">is_gzipped</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">is_valid</span><span class="p">:</span>
-<a name="cl-124"></a>            <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;The gzipped uploaded file contains inappropriate content&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-125"></a>            <span class="k">return</span>
-<a name="cl-126"></a>        <span class="k">elif</span> <span class="n">is_gzipped</span> <span class="ow">and</span> <span class="n">is_valid</span><span class="p">:</span>
-<a name="cl-127"></a>            <span class="k">if</span> <span class="n">link_data_only</span> <span class="o">==</span> <span class="s">&#39;copy_files&#39;</span><span class="p">:</span>
-<a name="cl-128"></a>                <span class="c"># We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format</span>
-<a name="cl-129"></a>                <span class="n">CHUNK_SIZE</span> <span class="o">=</span> <span class="mi">2</span><span class="o">**</span><span class="mi">20</span> <span class="c"># 1Mb   </span>
-<a name="cl-130"></a>                <span class="n">fd</span><span class="p">,</span> <span class="n">uncompressed</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkstemp</span><span class="p">(</span> <span class="n">prefix</span><span class="o">=</span><span class="s">&#39;data_id_</span><span class="si">%s</span><span class="s">_upload_gunzip_&#39;</span> <span class="o">%</span> <span class="n">dataset</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span> <span class="nb">dir</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span> <span class="n">output_path</span> <span class="p">),</span> <span class="n">text</span><span class="o">=</span><span class="bp">False</span> <span class="p">)</span>
-<a name="cl-131"></a>                <span class="n">gzipped_file</span> <span class="o">=</span> <span class="n">gzip</span><span class="o">.</span><span class="n">GzipFile</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="s">&#39;rb&#39;</span> <span class="p">)</span>
-<a name="cl-132"></a>                <span class="k">while</span> <span class="mi">1</span><span class="p">:</span>
-<a name="cl-133"></a>                    <span class="k">try</span><span class="p">:</span>
-<a name="cl-134"></a>                        <span class="n">chunk</span> <span class="o">=</span> <span class="n">gzipped_file</span><span class="o">.</span><span class="n">read</span><span class="p">(</span> <span class="n">CHUNK_SIZE</span> <span class="p">)</span>
-<a name="cl-135"></a>                    <span class="k">except</span> <span class="ne">IOError</span><span class="p">:</span>
-<a name="cl-136"></a>                        <span class="n">os</span><span class="o">.</span><span class="n">close</span><span class="p">(</span> <span class="n">fd</span> <span class="p">)</span>
-<a name="cl-137"></a>                        <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span> <span class="n">uncompressed</span> <span class="p">)</span>
-<a name="cl-138"></a>                        <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;Problem decompressing gzipped data&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-139"></a>                        <span class="k">return</span>
-<a name="cl-140"></a>                    <span class="k">if</span> <span class="ow">not</span> <span class="n">chunk</span><span class="p">:</span>
-<a name="cl-141"></a>                        <span class="k">break</span>
-<a name="cl-142"></a>                    <span class="n">os</span><span class="o">.</span><span class="n">write</span><span class="p">(</span> <span class="n">fd</span><span class="p">,</span> <span class="n">chunk</span> <span class="p">)</span>
-<a name="cl-143"></a>                <span class="n">os</span><span class="o">.</span><span class="n">close</span><span class="p">(</span> <span class="n">fd</span> <span class="p">)</span>
-<a name="cl-144"></a>                <span class="n">gzipped_file</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
-<a name="cl-145"></a>                <span class="c"># Replace the gzipped file with the decompressed file if it&#39;s safe to do so</span>
-<a name="cl-146"></a>                <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span> <span class="s">&#39;server_dir&#39;</span><span class="p">,</span> <span class="s">&#39;path_paste&#39;</span> <span class="p">)</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">in_place</span><span class="p">:</span>
-<a name="cl-147"></a>                    <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="n">uncompressed</span>
-<a name="cl-148"></a>                <span class="k">else</span><span class="p">:</span>
-<a name="cl-149"></a>                    <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span> <span class="n">uncompressed</span><span class="p">,</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-150"></a>                <span class="n">os</span><span class="o">.</span><span class="n">chmod</span><span class="p">(</span><span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="mo">0644</span><span class="p">)</span>
-<a name="cl-151"></a>            <span class="n">dataset</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span> <span class="s">&#39;.gz&#39;</span> <span class="p">)</span>
-<a name="cl-152"></a>            <span class="n">data_type</span> <span class="o">=</span> <span class="s">&#39;gzip&#39;</span>
-<a name="cl-153"></a>        <span class="k">if</span> <span class="ow">not</span> <span class="n">data_type</span> <span class="ow">and</span> <span class="n">bz2</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
-<a name="cl-154"></a>            <span class="c"># See if we have a bz2 file, much like gzip</span>
-<a name="cl-155"></a>            <span class="n">is_bzipped</span><span class="p">,</span> <span class="n">is_valid</span> <span class="o">=</span> <span class="n">check_bz2</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-156"></a>            <span class="k">if</span> <span class="n">is_bzipped</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">is_valid</span><span class="p">:</span>
-<a name="cl-157"></a>                <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;The gzipped uploaded file contains inappropriate content&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-158"></a>                <span class="k">return</span>
-<a name="cl-159"></a>            <span class="k">elif</span> <span class="n">is_bzipped</span> <span class="ow">and</span> <span class="n">is_valid</span><span class="p">:</span>
-<a name="cl-160"></a>                <span class="k">if</span> <span class="n">link_data_only</span> <span class="o">==</span> <span class="s">&#39;copy_files&#39;</span><span class="p">:</span>
-<a name="cl-161"></a>                    <span class="c"># We need to uncompress the temp_name file</span>
-<a name="cl-162"></a>                    <span class="n">CHUNK_SIZE</span> <span class="o">=</span> <span class="mi">2</span><span class="o">**</span><span class="mi">20</span> <span class="c"># 1Mb   </span>
-<a name="cl-163"></a>                    <span class="n">fd</span><span class="p">,</span> <span class="n">uncompressed</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkstemp</span><span class="p">(</span> <span class="n">prefix</span><span class="o">=</span><span class="s">&#39;data_id_</span><span class="si">%s</span><span class="s">_upload_bunzip2_&#39;</span> <span class="o">%</span> <span class="n">dataset</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span> <span class="nb">dir</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span> <span class="n">output_path</span> <span class="p">),</span> <span class="n">text</span><span class="o">=</span><span class="bp">False</span> <span class="p">)</span>
-<a name="cl-164"></a>                    <span class="n">bzipped_file</span> <span class="o">=</span> <span class="n">bz2</span><span class="o">.</span><span class="n">BZ2File</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="s">&#39;rb&#39;</span> <span class="p">)</span>
-<a name="cl-165"></a>                    <span class="k">while</span> <span class="mi">1</span><span class="p">:</span>
-<a name="cl-166"></a>                        <span class="k">try</span><span class="p">:</span>
-<a name="cl-167"></a>                            <span class="n">chunk</span> <span class="o">=</span> <span class="n">bzipped_file</span><span class="o">.</span><span class="n">read</span><span class="p">(</span> <span class="n">CHUNK_SIZE</span> <span class="p">)</span>
-<a name="cl-168"></a>                        <span class="k">except</span> <span class="ne">IOError</span><span class="p">:</span>
-<a name="cl-169"></a>                            <span class="n">os</span><span class="o">.</span><span class="n">close</span><span class="p">(</span> <span class="n">fd</span> <span class="p">)</span>
-<a name="cl-170"></a>                            <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span> <span class="n">uncompressed</span> <span class="p">)</span>
-<a name="cl-171"></a>                            <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;Problem decompressing bz2 compressed data&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-172"></a>                            <span class="k">return</span>
-<a name="cl-173"></a>                        <span class="k">if</span> <span class="ow">not</span> <span class="n">chunk</span><span class="p">:</span>
-<a name="cl-174"></a>                            <span class="k">break</span>
-<a name="cl-175"></a>                        <span class="n">os</span><span class="o">.</span><span class="n">write</span><span class="p">(</span> <span class="n">fd</span><span class="p">,</span> <span class="n">chunk</span> <span class="p">)</span>
-<a name="cl-176"></a>                    <span class="n">os</span><span class="o">.</span><span class="n">close</span><span class="p">(</span> <span class="n">fd</span> <span class="p">)</span>
-<a name="cl-177"></a>                    <span class="n">bzipped_file</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
-<a name="cl-178"></a>                    <span class="c"># Replace the bzipped file with the decompressed file if it&#39;s safe to do so</span>
-<a name="cl-179"></a>                    <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span> <span class="s">&#39;server_dir&#39;</span><span class="p">,</span> <span class="s">&#39;path_paste&#39;</span> <span class="p">)</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">in_place</span><span class="p">:</span>
-<a name="cl-180"></a>                        <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="n">uncompressed</span>
-<a name="cl-181"></a>                    <span class="k">else</span><span class="p">:</span>
-<a name="cl-182"></a>                        <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span> <span class="n">uncompressed</span><span class="p">,</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-183"></a>                    <span class="n">os</span><span class="o">.</span><span class="n">chmod</span><span class="p">(</span><span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="mo">0644</span><span class="p">)</span>
-<a name="cl-184"></a>                <span class="n">dataset</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span> <span class="s">&#39;.bz2&#39;</span> <span class="p">)</span>
-<a name="cl-185"></a>                <span class="n">data_type</span> <span class="o">=</span> <span class="s">&#39;bz2&#39;</span>
-<a name="cl-186"></a>        <span class="k">if</span> <span class="ow">not</span> <span class="n">data_type</span><span class="p">:</span>
-<a name="cl-187"></a>            <span class="c"># See if we have a zip archive</span>
-<a name="cl-188"></a>            <span class="n">is_zipped</span> <span class="o">=</span> <span class="n">check_zip</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-189"></a>            <span class="k">if</span> <span class="n">is_zipped</span><span class="p">:</span>
-<a name="cl-190"></a>                <span class="k">if</span> <span class="n">link_data_only</span> <span class="o">==</span> <span class="s">&#39;copy_files&#39;</span><span class="p">:</span>
-<a name="cl-191"></a>                    <span class="n">CHUNK_SIZE</span> <span class="o">=</span> <span class="mi">2</span><span class="o">**</span><span class="mi">20</span> <span class="c"># 1Mb</span>
-<a name="cl-192"></a>                    <span class="n">uncompressed</span> <span class="o">=</span> <span class="bp">None</span>
-<a name="cl-193"></a>                    <span class="n">uncompressed_name</span> <span class="o">=</span> <span class="bp">None</span>
-<a name="cl-194"></a>                    <span class="n">unzipped</span> <span class="o">=</span> <span class="bp">False</span>
-<a name="cl-195"></a>                    <span class="n">z</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-196"></a>                    <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">z</span><span class="o">.</span><span class="n">namelist</span><span class="p">():</span>
-<a name="cl-197"></a>                        <span class="k">if</span> <span class="n">name</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s">&#39;/&#39;</span><span class="p">):</span>
-<a name="cl-198"></a>                            <span class="k">continue</span>
-<a name="cl-199"></a>                        <span class="k">if</span> <span class="n">unzipped</span><span class="p">:</span>
-<a name="cl-200"></a>                            <span class="n">stdout</span> <span class="o">=</span> <span class="s">&#39;ZIP file contained more than one file, only the first file was added to Galaxy.&#39;</span>
-<a name="cl-201"></a>                            <span class="k">break</span>
-<a name="cl-202"></a>                        <span class="n">fd</span><span class="p">,</span> <span class="n">uncompressed</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkstemp</span><span class="p">(</span> <span class="n">prefix</span><span class="o">=</span><span class="s">&#39;data_id_</span><span class="si">%s</span><span class="s">_upload_zip_&#39;</span> <span class="o">%</span> <span class="n">dataset</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span> <span class="nb">dir</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span> <span class="n">output_path</span> <span class="p">),</span> <span class="n">text</span><span class="o">=</span><span class="bp">False</span> <span class="p">)</span>
-<a name="cl-203"></a>                        <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span> <span class="o">&gt;=</span> <span class="p">(</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">6</span> <span class="p">):</span>
-<a name="cl-204"></a>                            <span class="n">zipped_file</span> <span class="o">=</span> <span class="n">z</span><span class="o">.</span><span class="n">open</span><span class="p">(</span> <span class="n">name</span> <span class="p">)</span>
-<a name="cl-205"></a>                            <span class="k">while</span> <span class="mi">1</span><span class="p">:</span>
-<a name="cl-206"></a>                                <span class="k">try</span><span class="p">:</span>
-<a name="cl-207"></a>                                    <span class="n">chunk</span> <span class="o">=</span> <span class="n">zipped_file</span><span class="o">.</span><span class="n">read</span><span class="p">(</span> <span class="n">CHUNK_SIZE</span> <span class="p">)</span>
-<a name="cl-208"></a>                                <span class="k">except</span> <span class="ne">IOError</span><span class="p">:</span>
-<a name="cl-209"></a>                                    <span class="n">os</span><span class="o">.</span><span class="n">close</span><span class="p">(</span> <span class="n">fd</span> <span class="p">)</span>
-<a name="cl-210"></a>                                    <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span> <span class="n">uncompressed</span> <span class="p">)</span>
-<a name="cl-211"></a>                                    <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;Problem decompressing zipped data&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-212"></a>                                    <span class="k">return</span>
-<a name="cl-213"></a>                                <span class="k">if</span> <span class="ow">not</span> <span class="n">chunk</span><span class="p">:</span>
-<a name="cl-214"></a>                                    <span class="k">break</span>
-<a name="cl-215"></a>                                <span class="n">os</span><span class="o">.</span><span class="n">write</span><span class="p">(</span> <span class="n">fd</span><span class="p">,</span> <span class="n">chunk</span> <span class="p">)</span>
-<a name="cl-216"></a>                            <span class="n">os</span><span class="o">.</span><span class="n">close</span><span class="p">(</span> <span class="n">fd</span> <span class="p">)</span>
-<a name="cl-217"></a>                            <span class="n">zipped_file</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
-<a name="cl-218"></a>                            <span class="n">uncompressed_name</span> <span class="o">=</span> <span class="n">name</span>
-<a name="cl-219"></a>                            <span class="n">unzipped</span> <span class="o">=</span> <span class="bp">True</span>
-<a name="cl-220"></a>                        <span class="k">else</span><span class="p">:</span>
-<a name="cl-221"></a>                            <span class="c"># python &lt; 2.5 doesn&#39;t have a way to read members in chunks(!)</span>
-<a name="cl-222"></a>                            <span class="k">try</span><span class="p">:</span>
-<a name="cl-223"></a>                                <span class="n">outfile</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span> <span class="n">uncompressed</span><span class="p">,</span> <span class="s">&#39;wb&#39;</span> <span class="p">)</span>
-<a name="cl-224"></a>                                <span class="n">outfile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span> <span class="n">z</span><span class="o">.</span><span class="n">read</span><span class="p">(</span> <span class="n">name</span> <span class="p">)</span> <span class="p">)</span>
-<a name="cl-225"></a>                                <span class="n">outfile</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
-<a name="cl-226"></a>                                <span class="n">uncompressed_name</span> <span class="o">=</span> <span class="n">name</span>
-<a name="cl-227"></a>                                <span class="n">unzipped</span> <span class="o">=</span> <span class="bp">True</span>
-<a name="cl-228"></a>                            <span class="k">except</span> <span class="ne">IOError</span><span class="p">:</span>
-<a name="cl-229"></a>                                <span class="n">os</span><span class="o">.</span><span class="n">close</span><span class="p">(</span> <span class="n">fd</span> <span class="p">)</span>
-<a name="cl-230"></a>                                <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span> <span class="n">uncompressed</span> <span class="p">)</span>
-<a name="cl-231"></a>                                <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;Problem decompressing zipped data&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-232"></a>                                <span class="k">return</span>
-<a name="cl-233"></a>                    <span class="n">z</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
-<a name="cl-234"></a>                    <span class="c"># Replace the zipped file with the decompressed file if it&#39;s safe to do so</span>
-<a name="cl-235"></a>                    <span class="k">if</span> <span class="n">uncompressed</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
-<a name="cl-236"></a>                        <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span> <span class="s">&#39;server_dir&#39;</span><span class="p">,</span> <span class="s">&#39;path_paste&#39;</span> <span class="p">)</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">in_place</span><span class="p">:</span>
-<a name="cl-237"></a>                            <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="n">uncompressed</span>
-<a name="cl-238"></a>                        <span class="k">else</span><span class="p">:</span>
-<a name="cl-239"></a>                            <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span> <span class="n">uncompressed</span><span class="p">,</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">)</span>
-<a name="cl-240"></a>                        <span class="n">os</span><span class="o">.</span><span class="n">chmod</span><span class="p">(</span><span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="mo">0644</span><span class="p">)</span>
-<a name="cl-241"></a>                        <span class="n">dataset</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">uncompressed_name</span>
-<a name="cl-242"></a>                <span class="n">data_type</span> <span class="o">=</span> <span class="s">&#39;zip&#39;</span>
-<a name="cl-243"></a>        <span class="k">if</span> <span class="ow">not</span> <span class="n">data_type</span><span class="p">:</span>
-<a name="cl-244"></a>            <span class="k">if</span> <span class="n">check_binary</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">):</span>
-<a name="cl-245"></a>                <span class="c"># We have a binary dataset, but it is not Bam, Sff or Pdf</span>
-<a name="cl-246"></a>                <span class="n">data_type</span> <span class="o">=</span> <span class="s">&#39;binary&#39;</span>
-<a name="cl-247"></a>                <span class="c">#binary_ok = False</span>
-<a name="cl-248"></a>                <span class="n">parts</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span> <span class="s">&quot;.&quot;</span> <span class="p">)</span>
-<a name="cl-249"></a>                <span class="k">if</span> <span class="nb">len</span><span class="p">(</span> <span class="n">parts</span> <span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
-<a name="cl-250"></a>                    <span class="n">ext</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
-<a name="cl-251"></a>                    <span class="k">if</span> <span class="ow">not</span> <span class="n">Binary</span><span class="o">.</span><span class="n">is_ext_unsniffable</span><span class="p">(</span><span class="n">ext</span><span class="p">):</span>
-<a name="cl-252"></a>                        <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;The uploaded binary file contains inappropriate content&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-253"></a>                        <span class="k">return</span>
-<a name="cl-254"></a>                    <span class="k">elif</span> <span class="n">Binary</span><span class="o">.</span><span class="n">is_ext_unsniffable</span><span class="p">(</span><span class="n">ext</span><span class="p">)</span> <span class="ow">and</span> <span class="n">dataset</span><span class="o">.</span><span class="n">file_type</span> <span class="o">!=</span> <span class="n">ext</span><span class="p">:</span>
-<a name="cl-255"></a>                        <span class="n">err_msg</span> <span class="o">=</span> <span class="s">&quot;You must manually set the &#39;File Format&#39; to &#39;</span><span class="si">%s</span><span class="s">&#39; when uploading </span><span class="si">%s</span><span class="s"> files.&quot;</span> <span class="o">%</span> <span class="p">(</span> <span class="n">ext</span><span class="o">.</span><span class="n">capitalize</span><span class="p">(),</span> <span class="n">ext</span> <span class="p">)</span>
-<a name="cl-256"></a>                        <span class="n">file_err</span><span class="p">(</span> <span class="n">err_msg</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-257"></a>                        <span class="k">return</span>
-<a name="cl-258"></a>        <span class="k">if</span> <span class="ow">not</span> <span class="n">data_type</span><span class="p">:</span>
-<a name="cl-259"></a>            <span class="c"># We must have a text file</span>
-<a name="cl-260"></a>            <span class="k">if</span> <span class="n">check_html</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">):</span>
-<a name="cl-261"></a>                <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;The uploaded file contains inappropriate HTML content&#39;</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-262"></a>                <span class="k">return</span>
-<a name="cl-263"></a>        <span class="k">if</span> <span class="n">data_type</span> <span class="o">!=</span> <span class="s">&#39;binary&#39;</span><span class="p">:</span>
-<a name="cl-264"></a>            <span class="k">if</span> <span class="n">link_data_only</span> <span class="o">==</span> <span class="s">&#39;copy_files&#39;</span><span class="p">:</span>
-<a name="cl-265"></a>                <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span> <span class="s">&#39;server_dir&#39;</span><span class="p">,</span> <span class="s">&#39;path_paste&#39;</span> <span class="p">)</span> <span class="ow">and</span> <span class="n">data_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span> <span class="s">&#39;gzip&#39;</span><span class="p">,</span> <span class="s">&#39;bz2&#39;</span><span class="p">,</span> <span class="s">&#39;zip&#39;</span> <span class="p">]:</span>
-<a name="cl-266"></a>                    <span class="n">in_place</span> <span class="o">=</span> <span class="bp">False</span>
-<a name="cl-267"></a>                <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">space_to_tab</span><span class="p">:</span>
-<a name="cl-268"></a>                    <span class="n">line_count</span><span class="p">,</span> <span class="n">converted_path</span> <span class="o">=</span> <span class="n">sniff</span><span class="o">.</span><span class="n">convert_newlines_sep2tabs</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">in_place</span><span class="o">=</span><span class="n">in_place</span> <span class="p">)</span>
-<a name="cl-269"></a>                <span class="k">else</span><span class="p">:</span>
-<a name="cl-270"></a>                    <span class="n">line_count</span><span class="p">,</span> <span class="n">converted_path</span> <span class="o">=</span> <span class="n">sniff</span><span class="o">.</span><span class="n">convert_newlines</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">in_place</span><span class="o">=</span><span class="n">in_place</span> <span class="p">)</span>
-<a name="cl-271"></a>            <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">file_type</span> <span class="o">==</span> <span class="s">&#39;auto&#39;</span><span class="p">:</span>
-<a name="cl-272"></a>                <span class="n">ext</span> <span class="o">=</span> <span class="n">sniff</span><span class="o">.</span><span class="n">guess_ext</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">registry</span><span class="o">.</span><span class="n">sniff_order</span> <span class="p">)</span>
-<a name="cl-273"></a>            <span class="k">else</span><span class="p">:</span>
-<a name="cl-274"></a>                <span class="n">ext</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">file_type</span>
-<a name="cl-275"></a>            <span class="n">data_type</span> <span class="o">=</span> <span class="n">ext</span>
-<a name="cl-276"></a>    <span class="c"># Save job info for the framework</span>
-<a name="cl-277"></a>    <span class="k">if</span> <span class="n">ext</span> <span class="o">==</span> <span class="s">&#39;auto&#39;</span> <span class="ow">and</span> <span class="n">dataset</span><span class="o">.</span><span class="n">ext</span><span class="p">:</span>
-<a name="cl-278"></a>        <span class="n">ext</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">ext</span>
-<a name="cl-279"></a>    <span class="k">if</span> <span class="n">ext</span> <span class="o">==</span> <span class="s">&#39;auto&#39;</span><span class="p">:</span>
-<a name="cl-280"></a>        <span class="n">ext</span> <span class="o">=</span> <span class="s">&#39;data&#39;</span>
-<a name="cl-281"></a>    <span class="n">datatype</span> <span class="o">=</span> <span class="n">registry</span><span class="o">.</span><span class="n">get_datatype_by_extension</span><span class="p">(</span> <span class="n">ext</span> <span class="p">)</span>
-<a name="cl-282"></a>    <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span> <span class="s">&#39;server_dir&#39;</span><span class="p">,</span> <span class="s">&#39;path_paste&#39;</span> <span class="p">)</span> <span class="ow">and</span> <span class="n">link_data_only</span> <span class="o">==</span> <span class="s">&#39;link_to_files&#39;</span><span class="p">:</span>
-<a name="cl-283"></a>        <span class="c"># Never alter a file that will not be copied to Galaxy&#39;s local file store.</span>
-<a name="cl-284"></a>        <span class="k">if</span> <span class="n">datatype</span><span class="o">.</span><span class="n">dataset_content_needs_grooming</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="p">):</span>
-<a name="cl-285"></a>            <span class="n">err_msg</span> <span class="o">=</span> <span class="s">&#39;The uploaded files need grooming, so change your &lt;b&gt;Copy data into Galaxy?&lt;/b&gt; selection to be &#39;</span> <span class="o">+</span> \
-<a name="cl-286"></a>                <span class="s">&#39;&lt;b&gt;Copy files into Galaxy&lt;/b&gt; instead of &lt;b&gt;Link to files without copying into Galaxy&lt;/b&gt; so grooming can be performed.&#39;</span>
-<a name="cl-287"></a>            <span class="n">file_err</span><span class="p">(</span> <span class="n">err_msg</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-288"></a>            <span class="k">return</span>
-<a name="cl-289"></a>    <span class="k">if</span> <span class="n">link_data_only</span> <span class="o">==</span> <span class="s">&#39;copy_files&#39;</span> <span class="ow">and</span> <span class="n">dataset</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span> <span class="s">&#39;server_dir&#39;</span><span class="p">,</span> <span class="s">&#39;path_paste&#39;</span> <span class="p">)</span> <span class="ow">and</span> <span class="n">data_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span> <span class="s">&#39;gzip&#39;</span><span class="p">,</span> <span class="s">&#39;bz2&#39;</span><span class="p">,</span> <span class="s">&#39;zip&#39;</span> <span class="p">]:</span>
-<a name="cl-290"></a>        <span class="c"># Move the dataset to its &quot;real&quot; path</span>
-<a name="cl-291"></a>        <span class="k">if</span> <span class="n">converted_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
-<a name="cl-292"></a>            <span class="n">shutil</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span> <span class="n">converted_path</span><span class="p">,</span> <span class="n">output_path</span> <span class="p">)</span>
-<a name="cl-293"></a>            <span class="k">try</span><span class="p">:</span>
-<a name="cl-294"></a>                <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span> <span class="n">converted_path</span> <span class="p">)</span>
-<a name="cl-295"></a>            <span class="k">except</span><span class="p">:</span>
-<a name="cl-296"></a>                <span class="k">pass</span>
-<a name="cl-297"></a>        <span class="k">else</span><span class="p">:</span>
-<a name="cl-298"></a>            <span class="c"># This should not happen, but it&#39;s here just in case</span>
-<a name="cl-299"></a>            <span class="n">shutil</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">output_path</span> <span class="p">)</span>
-<a name="cl-300"></a>    <span class="k">elif</span> <span class="n">link_data_only</span> <span class="o">==</span> <span class="s">&#39;copy_files&#39;</span><span class="p">:</span>
-<a name="cl-301"></a>        <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">path</span><span class="p">,</span> <span class="n">output_path</span> <span class="p">)</span>
-<a name="cl-302"></a>    <span class="c"># Write the job info</span>
-<a name="cl-303"></a>    <span class="n">stdout</span> <span class="o">=</span> <span class="n">stdout</span> <span class="ow">or</span> <span class="s">&#39;uploaded </span><span class="si">%s</span><span class="s"> file&#39;</span> <span class="o">%</span> <span class="n">data_type</span>
-<a name="cl-304"></a>    <span class="n">info</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span> <span class="nb">type</span> <span class="o">=</span> <span class="s">&#39;dataset&#39;</span><span class="p">,</span>
-<a name="cl-305"></a>                 <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span>
-<a name="cl-306"></a>                 <span class="n">ext</span> <span class="o">=</span> <span class="n">ext</span><span class="p">,</span>
-<a name="cl-307"></a>                 <span class="n">stdout</span> <span class="o">=</span> <span class="n">stdout</span><span class="p">,</span>
-<a name="cl-308"></a>                 <span class="n">name</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
-<a name="cl-309"></a>                 <span class="n">line_count</span> <span class="o">=</span> <span class="n">line_count</span> <span class="p">)</span>
-<a name="cl-310"></a>    <span class="n">json_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span> <span class="n">to_json_string</span><span class="p">(</span> <span class="n">info</span> <span class="p">)</span> <span class="o">+</span> <span class="s">&quot;</span><span class="se">\n</span><span class="s">&quot;</span> <span class="p">)</span>
-<a name="cl-311"></a>
-<a name="cl-312"></a>    <span class="k">if</span> <span class="n">link_data_only</span> <span class="o">==</span> <span class="s">&#39;copy_files&#39;</span> <span class="ow">and</span> <span class="n">datatype</span><span class="o">.</span><span class="n">dataset_content_needs_grooming</span><span class="p">(</span> <span class="n">output_path</span> <span class="p">):</span>
-<a name="cl-313"></a>        <span class="c"># Groom the dataset content if necessary</span>
-<a name="cl-314"></a>        <span class="n">datatype</span><span class="o">.</span><span class="n">groom_dataset_content</span><span class="p">(</span> <span class="n">output_path</span> <span class="p">)</span>
-<a name="cl-315"></a>
-<a name="cl-316"></a><span class="k">def</span> <span class="nf">add_composite_file</span><span class="p">(</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">registry</span><span class="p">,</span> <span class="n">json_file</span><span class="p">,</span> <span class="n">output_path</span><span class="p">,</span> <span class="n">files_path</span> <span class="p">):</span>
-<a name="cl-317"></a>        <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">composite_files</span><span class="p">:</span>
-<a name="cl-318"></a>            <span class="n">os</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span> <span class="n">files_path</span> <span class="p">)</span>
-<a name="cl-319"></a>            <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">dataset</span><span class="o">.</span><span class="n">composite_files</span><span class="o">.</span><span class="n">iteritems</span><span class="p">():</span>
-<a name="cl-320"></a>                <span class="n">value</span> <span class="o">=</span> <span class="n">util</span><span class="o">.</span><span class="n">bunch</span><span class="o">.</span><span class="n">Bunch</span><span class="p">(</span> <span class="o">**</span><span class="n">value</span> <span class="p">)</span>
-<a name="cl-321"></a>                <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">composite_file_paths</span><span class="p">[</span> <span class="n">value</span><span class="o">.</span><span class="n">name</span> <span class="p">]</span> <span class="ow">is</span> <span class="bp">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">value</span><span class="o">.</span><span class="n">optional</span><span class="p">:</span>
-<a name="cl-322"></a>                    <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;A required composite data file was not provided (</span><span class="si">%s</span><span class="s">)&#39;</span> <span class="o">%</span> <span class="n">name</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-323"></a>                    <span class="k">break</span>
-<a name="cl-324"></a>                <span class="k">elif</span> <span class="n">dataset</span><span class="o">.</span><span class="n">composite_file_paths</span><span class="p">[</span><span class="n">value</span><span class="o">.</span><span class="n">name</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
-<a name="cl-325"></a>                    <span class="n">dp</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">composite_file_paths</span><span class="p">[</span><span class="n">value</span><span class="o">.</span><span class="n">name</span><span class="p">][</span> <span class="s">&#39;path&#39;</span> <span class="p">]</span>
-<a name="cl-326"></a>                    <span class="n">isurl</span> <span class="o">=</span> <span class="n">dp</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">&#39;://&#39;</span><span class="p">)</span> <span class="o">&lt;&gt;</span> <span class="o">-</span><span class="mi">1</span> <span class="c"># todo fixme</span>
-<a name="cl-327"></a>                    <span class="k">if</span> <span class="n">isurl</span><span class="p">:</span>
-<a name="cl-328"></a>                       <span class="k">try</span><span class="p">:</span>
-<a name="cl-329"></a>                           <span class="n">temp_name</span><span class="p">,</span> <span class="n">dataset</span><span class="o">.</span><span class="n">is_multi_byte</span> <span class="o">=</span> <span class="n">sniff</span><span class="o">.</span><span class="n">stream_to_file</span><span class="p">(</span> <span class="n">urllib</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span> <span class="n">dp</span> <span class="p">),</span> <span class="n">prefix</span><span class="o">=</span><span class="s">&#39;url_paste&#39;</span> <span class="p">)</span>
-<a name="cl-330"></a>                       <span class="k">except</span> <span class="ne">Exception</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span>
-<a name="cl-331"></a>                           <span class="n">file_err</span><span class="p">(</span> <span class="s">&#39;Unable to fetch </span><span class="si">%s</span><span class="se">\n</span><span class="si">%s</span><span class="s">&#39;</span> <span class="o">%</span> <span class="p">(</span> <span class="n">dp</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span> <span class="n">e</span> <span class="p">)</span> <span class="p">),</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">json_file</span> <span class="p">)</span>
-<a name="cl-332"></a>                           <span class="k">return</span>
-<a name="cl-333"></a>                       <span class="n">dataset</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="n">temp_name</span>
-<a name="cl-334"></a>                       <span class="n">dp</span> <span class="o">=</span> <span class="n">temp_name</span>
-<a name="cl-335"></a>                    <span class="k">if</span> <span class="ow">not</span> <span class="n">value</span><span class="o">.</span><span class="n">is_binary</span><span class="p">:</span>
-<a name="cl-336"></a>                        <span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">composite_file_paths</span><span class="p">[</span> <span class="n">value</span><span class="o">.</span><span class="n">name</span> <span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> <span class="s">&#39;space_to_tab&#39;</span><span class="p">,</span> <span class="n">value</span><span class="o">.</span><span class="n">space_to_tab</span> <span class="p">):</span>
-<a name="cl-337"></a>                            <span class="n">sniff</span><span class="o">.</span><span class="n">convert_newlines_sep2tabs</span><span class="p">(</span> <span class="n">dp</span> <span class="p">)</span>
-<a name="cl-338"></a>                        <span class="k">else</span><span class="p">:</span>
-<a name="cl-339"></a>                            <span class="n">sniff</span><span class="o">.</span><span class="n">convert_newlines</span><span class="p">(</span> <span class="n">dp</span> <span class="p">)</span>
-<a name="cl-340"></a>                    <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span> <span class="n">dp</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span> <span class="n">files_path</span><span class="p">,</span> <span class="n">name</span> <span class="p">)</span> <span class="p">)</span>
-<a name="cl-341"></a>        <span class="c"># Move the dataset to its &quot;real&quot; path</span>
-<a name="cl-342"></a>        <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span> <span class="n">dataset</span><span class="o">.</span><span class="n">primary_file</span><span class="p">,</span> <span class="n">output_path</span> <span class="p">)</span>
-<a name="cl-343"></a>        <span class="c"># Write the job info</span>
-<a name="cl-344"></a>        <span class="n">info</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span> <span class="nb">type</span> <span class="o">=</span> <span class="s">&#39;dataset&#39;</span><span class="p">,</span>
-<a name="cl-345"></a>                     <span class="n">dataset_id</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">dataset_id</span><span class="p">,</span>
-&l