Commits

Daniel Blankenberg committed 3189a1b

Add sentry_dsn to Tool Shed config.py' lib/galaxy/webapps/community/config.py

  • Participants
  • Parent commits d7f4d3a

Comments (0)

Files changed (15)

File datatypes_conf.xml.sample

     <datatype extension="bowtie_base_index" type="galaxy.datatypes.ngsindex:BowtieBaseIndex" mimetype="text/html" display_in_upload="False"/>
     <datatype extension="csfasta" type="galaxy.datatypes.sequence:csFasta" display_in_upload="true"/>
     <datatype extension="data" type="galaxy.datatypes.data:Data" mimetype="application/octet-stream" max_optional_metadata_filesize="1048576" />
+    <datatype extension="data_manager_json" type="galaxy.datatypes.data:Text" mimetype="application/json" subclass="True" display_in_upload="False"/>
     <datatype extension="fasta" type="galaxy.datatypes.sequence:Fasta" display_in_upload="true">
       <converter file="fasta_to_tabular_converter.xml" target_datatype="tabular"/>
       <converter file="fasta_to_bowtie_base_index_converter.xml" target_datatype="bowtie_base_index"/>

File lib/galaxy/app.py

 from galaxy.tools.genome_index import load_genome_index_tools
 from galaxy.sample_tracking import external_service_types
 from galaxy.openid.providers import OpenIDProviders
+from galaxy.tools.data_manager.manager import DataManagers
 
 class UniverseApplication( object ):
     """Encapsulates the state of a Universe application"""
         self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self )
         # Search support for tools
         self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox )
+        #datamanager
+        self.data_managers = DataManagers( self )
         # If enabled, poll respective tool sheds to see if updates are available for any installed tool shed repositories.
         if self.config.get_bool( 'enable_tool_shed_check', False ):
             from tool_shed import update_manager

File lib/galaxy/config.py

         except:
             self.hours_between_check = 12
         self.update_integrated_tool_panel = kwargs.get( "update_integrated_tool_panel", True )
+        self.enable_data_manager_user_view = string_as_bool( kwargs.get( "enable_data_manager_user_view", "False" ) )
+        self.data_manager_config_file = resolve_path( kwargs.get(' data_manager_config_file', 'data_manager_conf.xml' ), self.root )
+        self.data_manager_move_files = string_as_bool( kwargs.get( "data_manager_move_files", "False" ) )
+        self.galaxy_data_manager_data_path = kwargs.get( 'galaxy_data_manager_data_dir',  self.tool_data_path )
         self.tool_secret = kwargs.get( "tool_secret", "" )
         self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
         self.set_metadata_externally = string_as_bool( kwargs.get( "set_metadata_externally", "False" ) )

File lib/galaxy/jobs/runners/local.py

         log.debug( "%d workers ready", nworkers )
 
     def run_next( self ):
-        """Run the next job, waiting until one is available if neccesary"""
+        """Run the next job, waiting until one is available if necessary"""
         while 1:
             job_wrapper = self.queue.get()
             if job_wrapper is self.STOP_SIGNAL:

File lib/galaxy/model/__init__.py

     def set_item( self, visualization ):
         self.visualization = visualization
 
+#Data Manager Classes
+class DataManagerHistoryAssociation( object ):
+    def __init__( self, id=None, history=None, user=None ):
+        self.id = id
+        self.history = history
+        self.user = user
+
+class DataManagerJobAssociation( object ):
+    def __init__( self, id=None, job=None, data_manager_id=None ):
+        self.id = id
+        self.job = job
+        self.data_manager_id = data_manager_id
+#end of Data Manager Classes
+
 class UserPreference ( object ):
     def __init__( self, name=None, value=None ):
         self.name = name

File lib/galaxy/model/mapping.py

     Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )
     )
     
+#Data Manager tables
+DataManagerHistoryAssociation.table = Table( "data_manager_history_association", metadata,
+    Column( "id", Integer, primary_key=True),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
+    Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )
+    )
+
+DataManagerJobAssociation.table = Table( "data_manager_job_association", metadata,
+    Column( "id", Integer, primary_key=True),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
+    Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
+    Column( "data_manager_id", TEXT, index=True )
+    )
+
 # Tagging tables.
 
 Tag.table = Table( "tag", metadata,
     properties=dict( visualization=relation( Visualization ), user=relation( User ) )
                     )
 
+#Data Manager tables
+assign_mapper( context, DataManagerHistoryAssociation, DataManagerHistoryAssociation.table,
+    properties=dict( history=relation( History ),
+                     user=relation( User, backref='data_manager_histories' )
+                    )
+              )
+
+assign_mapper( context, DataManagerJobAssociation, DataManagerJobAssociation.table,
+    properties=dict( job=relation( Job, backref=backref('data_manager_association', uselist=False ), uselist=False ) )
+              )
+
 # User tables.
                     
 assign_mapper( context, UserPreference, UserPreference.table, 

File lib/galaxy/tools/__init__.py

 pkg_resources.require( "simplejson" )
 pkg_resources.require( "Mako" )
 
-import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess, random, math, traceback, re
+import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess, random, math, traceback, re, pipes
 import simplejson
 import binascii
 from mako.template import Template
 from galaxy.util.expressions import ExpressionContext
 from galaxy.tools.test import ToolTestBuilder
 from galaxy.tools.actions import DefaultToolAction
+from galaxy.tools.actions.data_manager import DataManagerToolAction
 from galaxy.tools.deps import DependencyManager
 from galaxy.model import directory_hash_id
 from galaxy.model.orm import *
         # In-memory dictionary that defines the layout of the tool panel.
         self.tool_panel = odict()
         self.index = 0
+        self.data_manager_tools = odict()
         # File that contains the XML section and tool tags from all tool panel config files integrated into a
         # single file that defines the tool panel layout.  This file can be changed by the Galaxy administrator
         # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel.
             self.integrated_tool_panel[ key ] = integrated_section
         else:
             self.integrated_tool_panel.insert( index, key, integrated_section )
-    def load_tool( self, config_file, guid=None ):
+    def load_tool( self, config_file, guid=None, **kwds ):
         """Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
         # Parse XML configuration file and get the root element
         tree = util.parse_xml( config_file )
             ToolClass = tool_types.get( root.get( 'tool_type' ) )
         else:
             ToolClass = Tool
-        return ToolClass( config_file, root, self.app, guid=guid )
+        return ToolClass( config_file, root, self.app, guid=guid, **kwds )
     def reload_tool_by_id( self, tool_id ):
         """
         Attempt to reload the tool identified by 'tool_id', if successful
     """
     
     tool_type = 'default'
+    default_tool_action = DefaultToolAction
     
     def __init__( self, config_file, root, app, guid=None ):
         """Load a tool from the config named by `config_file`"""
         # Action
         action_elem = root.find( "action" )
         if action_elem is None:
-            self.tool_action = DefaultToolAction()
+            self.tool_action = self.default_tool_action()
         else:
             module = action_elem.get( 'module' )
             cls = action_elem.get( 'class' )
             temp_file_path = os.path.join( job_working_directory, "dataset_%s_files" % ( hda.dataset.id ) )
             try:
                 a_files = os.listdir( temp_file_path )
+                print 'a_files',a_files
                 if len( a_files ) > 0:
                     for f in a_files:
+                        print 'f', f
                         self.app.object_store.update_from_file(hda.dataset,
                             extra_dir="dataset_%d_files" % hda.dataset.id, 
                             alt_name = f,
                             file_name = os.path.join(temp_file_path, f),
-                            create = True)
+                            create = True,
+                            preserve_symlinks = True )
                     # Clean up after being handled by object store. 
                     # FIXME: If the object (e.g., S3) becomes async, this will 
                     # cause issues so add it to the object store functionality?
+                    print 'before rmtree'
                     shutil.rmtree(temp_file_path)
-            except:
+                    print 'after rm tree'
+            except Exception, e:
+                log.debug( "Error in collect_associated_files: %s" % ( e ) )
                 continue
     def collect_child_datasets( self, output, job_working_directory ):
         """
         
         return tool_dict
 
-class DataSourceTool( Tool ):
+    def get_default_history_by_trans( self, trans, create=False ):
+        return trans.get_history( create=create )
+
+
+class OutputParameterJSONTool( Tool ):
+    """
+    Alternate implementation of Tool that provides parameters and other values 
+    JSONified within the contents of an output dataset
+    """
+    tool_type = 'output_parameter_json'
+    def _prepare_json_list( self, param_list ):
+        rval = []
+        for value in param_list:
+            if isinstance( value, dict ):
+                rval.append( self._prepare_json_param_dict( value ) )
+            elif isinstance( value, list ):
+                rval.append( self._prepare_json_list( value ) )
+            else:
+                rval.append( str( value ) )
+        return rval
+    def _prepare_json_param_dict( self, param_dict ):
+        rval = {}
+        for key, value in param_dict.iteritems():
+            if isinstance( value, dict ):
+                rval[ key ] = self._prepare_json_param_dict( value )
+            elif isinstance( value, list ):
+                rval[ key ] = self._prepare_json_list( value )
+            else:
+                rval[ key ] = str( value )
+        return rval
+    def exec_before_job( self, app, inp_data, out_data, param_dict=None ):
+        if param_dict is None:
+            param_dict = {}        
+        json_params = {}
+        json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) #it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones?
+        json_params[ 'output_data' ] = []
+        json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE )
+        json_filename = None
+        for i, ( out_name, data ) in enumerate( out_data.iteritems() ):
+            #use wrapped dataset to access certain values 
+            wrapped_data = param_dict.get( out_name )
+            #allow multiple files to be created
+            file_name = str( wrapped_data )
+            extra_files_path = str( wrapped_data.files_path )
+            data_dict = dict( out_data_name = out_name,
+                              ext = data.ext,
+                              dataset_id = data.dataset.id,
+                              hda_id = data.id,
+                              file_name = file_name,
+                              extra_files_path = extra_files_path )
+            json_params[ 'output_data' ].append( data_dict )
+            if json_filename is None:
+                json_filename = file_name
+        out = open( json_filename, 'w' )
+        out.write( simplejson.dumps( json_params ) )
+        out.close()
+
+class DataSourceTool( OutputParameterJSONTool ):
     """
     Alternate implementation of Tool for data_source tools -- those that 
     allow the user to query and extract data from another web site.
     def _build_GALAXY_URL_parameter( self ):
         return ToolParameter.build( self, ElementTree.XML( '<param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=%s" />' % self.id ) )
     def parse_inputs( self, root ):
-        Tool.parse_inputs( self, root )
+        super( DataSourceTool, self ).parse_inputs( root )
         if 'GALAXY_URL' not in self.inputs:
             self.inputs[ 'GALAXY_URL' ] = self._build_GALAXY_URL_parameter()
-    def _prepare_datasource_json_list( self, param_list ):
-        rval = []
-        for value in param_list:
-            if isinstance( value, dict ):
-                rval.append( self._prepare_datasource_json_param_dict( value ) )
-            elif isinstance( value, list ):
-                rval.append( self._prepare_datasource_json_list( value ) )
-            else:
-                rval.append( str( value ) )
-        return rval
-    def _prepare_datasource_json_param_dict( self, param_dict ):
-        rval = {}
-        for key, value in param_dict.iteritems():
-            if isinstance( value, dict ):
-                rval[ key ] = self._prepare_datasource_json_param_dict( value )
-            elif isinstance( value, list ):
-                rval[ key ] = self._prepare_datasource_json_list( value )
-            else:
-                rval[ key ] = str( value )
-        return rval
+            self.inputs_by_page[0][ 'GALAXY_URL' ] = self.inputs[ 'GALAXY_URL' ]
     def exec_before_job( self, app, inp_data, out_data, param_dict=None ):
         if param_dict is None:
             param_dict = {}
         name = param_dict.get( 'name' )
         
         json_params = {}
-        json_params[ 'param_dict' ] = self._prepare_datasource_json_param_dict( param_dict ) #it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones?
+        json_params[ 'param_dict' ] = self._prepare_json_param_dict( param_dict ) #it would probably be better to store the original incoming parameters here, instead of the Galaxy modified ones?
         json_params[ 'output_data' ] = []
         json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=jobs.TOOL_PROVIDED_JOB_METADATA_FILE )
         json_filename = None
 class GenomeIndexTool( Tool ):
     tool_type = 'index_genome'
 
+class DataManagerTool( OutputParameterJSONTool ):
+    tool_type = 'manage_data'
+    default_tool_action = DataManagerToolAction
+    
+    def __init__( self, config_file, root, app, guid=None, data_manager_id=None, **kwds ):
+        self.data_manager_id = data_manager_id
+        super( DataManagerTool, self ).__init__( config_file, root, app, guid=guid, **kwds )
+        if self.data_manager_id is None:
+            self.data_manager_id = self.id
+    
+    #def parse_inputs( self, root ):
+    #    super( DataManagerTool, self ).parse_inputs( root )
+    #    '''
+    #    if '__GALAXY_MOVE_OUTPUT_FILES__' not in self.inputs:
+    #        self.inputs[ '__GALAXY_MOVE_OUTPUT_FILES__' ] = ToolParameter.build( self, ElementTree.XML( '<param name="__GALAXY_MOVE_OUTPUT_FILES__" label="Move created data to cache destination" type="boolean" truevalue="MOVE" falsevalue="DO_NOT_MOVE" checked="%s" />' % self.app.config.data_manager_move_files ) )
+    #        print 'self.inputs_by_page',self.inputs_by_page
+    #        self.inputs_by_page[0][ '__GALAXY_MOVE_OUTPUT_FILES__' ] = self.inputs[ '__GALAXY_MOVE_OUTPUT_FILES__' ]
+    #    print 'self.inputs', self.inputs
+    #    '''
+    #    #self.inputs[ '__DATA_MANAGER_ID__' ] = ToolParameter.build( self, ElementTree.XML( '<param name="__DATA_MANAGER_ID__" type="hidden" value="%s" />' % ( self.data_manager_id ) ) )
+    #    #self.inputs_by_page[0][ '__DATA_MANAGER_ID__' ] = self.inputs[ '__DATA_MANAGER_ID__' ]
+    
+    def exec_after_process( self, app, inp_data, out_data, param_dict, job = None, **kwds ):
+        #run original exec_after_process
+        super( DataManagerTool, self ).exec_after_process( app, inp_data, out_data, param_dict, job = job, **kwds )
+        #process results of tool
+        print 'exect after', self.id
+        print 'inp_data', inp_data
+        print 'out_data', out_data
+        print 'param_dict', param_dict
+        print 'job', job, job.state
+        if job and job.state == job.states.ERROR:
+            return
+        #print 'data_manager.output_ref',data_manager.output_ref
+        #data_manager = self.app.data_managers.get( self.id, None ) #fix me to not only use tool ID!
+        data_manager_id = job.data_manager_association.data_manager_id
+        data_manager = self.app.data_managers.get( data_manager_id, None )
+        #TODO: need to be able to handle using a data manager tool for more than one manager
+        #manager id is currently same as tool id
+        assert data_manager is not None, "Invalid data manager (%s) requested. It may have been removed before the job completed." % ( data_manager_id )
+        data_manager_dicts = {}
+        data_manager_dict = {}
+        #TODO: fix this merging below
+        for output_name, output_dataset in out_data.iteritems():
+            try:
+                output_dict = simplejson.loads( open( output_dataset.file_name ).read() )
+            except Exception, e:
+                log.warning( 'Error reading DataManagerTool json for "%s": %s'  % ( output_name, e ) )
+                continue
+            data_manager_dicts[ output_name ] = output_dict
+            print 'data_manager_dicts', data_manager_dicts
+            for key, value in output_dict.iteritems():
+                if key not in data_manager_dict:
+                    data_manager_dict[ key ] = {}
+                print 'key', key
+                print ' data_manager_dict[ key ]',  data_manager_dict[ key ]
+                print 'value', value
+                data_manager_dict[ key ].update( value )
+            data_manager_dict.update( output_dict )
+                
+        print 'data_manager_dicts',data_manager_dicts
+        print 'data_manager_dict', data_manager_dict
+        data_tables_dict = data_manager_dict.get( 'data_tables', {} )
+        #for data_table_name, data_table_values in data_tables_dict.iteritems():
+        for data_table_name, data_table_columns in data_manager.data_tables.iteritems():
+            print 'data_table_name', data_table_name
+            data_table_values = data_tables_dict.pop( data_table_name, None ) #data_tables_dict.get( data_table_name, [] )
+            if not data_table_values:
+                log.warning( 'No values for data table "%s" were returned by the data manager "%s".' % ( data_table_name, data_manager.id ) )
+                continue #next data table
+            data_table = app.tool_data_tables.get( data_table_name, None )
+            if data_table is None:
+                log.error( 'The data manager "%s" returned an unknown data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( data_manager.id, data_table_name, data_table_values, data_table_name, 'tool_data_table_conf.xml' ) )
+                continue #next table name
+            output_ref_values = {}
+            if data_table_name in data_manager.output_ref_by_data_table:
+                for data_table_column, output_ref in data_manager.output_ref_by_data_table[ data_table_name ].iteritems():
+                    output_ref_dataset = out_data.get( output_ref, None )
+                    assert output_ref_dataset is not None, "Referenced output was not found."
+                    output_ref_values[ data_table_column ] = output_ref_dataset
+            print 'output_ref_values', output_ref_values
+            
+            final_data_table_values = []
+            if not isinstance( data_table_values, list ):
+                data_table_values = [ data_table_values ]
+            columns = data_table.get_column_name_list()
+            
+            try:
+                data_table_fh = open( data_table.filename, 'r+b' )
+            except IOError, e:
+                log.warning( 'Error opening data table file (%s) with r+b, assuming file does not exist and will open as wb: %s' % ( data_table.filename, e ) )
+                data_table_fh = open( data_table.filename, 'wb' )
+            if os.stat( data_table.filename )[6] != 0:
+                # ensure last existing line ends with new line
+                data_table_fh.seek( -1, 2 ) #last char in file
+                last_char = data_table_fh.read()
+                if last_char not in [ '\n', '\r' ]:
+                    data_table_fh.write( '\n' )
+            for data_table_row in data_table_values:
+                data_table_value = dict( **data_table_row ) #keep original values here
+                for name, value in data_table_row.iteritems(): #FIXME: need to loop through here based upon order listed in data_manager config
+                    if name in output_ref_values:
+                        #TODO: Allow moving!
+                        #if param_dict[ '__GALAXY_MOVE_OUTPUT_FILES__' ]:
+                        #    #FIXME: allow moving
+                        #    log.error( "\n\nShould be moving output files directory, but not implemented yet.\n" )
+                        #    base_path = output_ref_values[ name ].extra_files_path    
+                        #else:
+                        #    base_path = output_ref_values[ name ].extra_files_path
+                        moved = data_manager.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value )
+                        print 'moved', moved #should we always move?
+                        data_table_value[ name ] = data_manager.process_value_translation( data_table_name, name, **data_table_value )
+                final_data_table_values.append( data_table_value )
+                fields = []
+                for column_name in columns:
+                    if column_name is None or column_name not in data_table_value:
+                        fields.append( data_table.get_empty_field_by_name( column_name ) )
+                    else:
+                        fields.append( data_table_value[ column_name ] )
+                print 'fields', fields
+                #should we add a comment to file about automatically generated value here?
+                data_table_fh.write( "%s\n" % ( data_table.separator.join( self._replace_field_separators( fields, separator=data_table.separator ) ) ) ) #write out fields to disk
+                data_table.data.append( fields ) #add fields to loaded data table
+            print 'final_data_table_values', final_data_table_values
+            print 'data_table.data', data_table.data
+            data_table_fh.close()
+        for data_table_name, data_table_values in data_tables_dict.iteritems():
+            #tool returned extra data table entries, but data table was not declared in data manager
+            #do not add these values, but do provide messages
+            log.warning( 'The data manager "%s" returned an undeclared data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( data_manager.id, data_table_name, data_table_values, data_table_name, self.app.data_managers.filename ) )
+        
+    def _replace_field_separators( self, fields, separator="\t", replace=None, comment_char=None ):
+        #make sure none of the fields contain separator
+        #make sure separator replace is different from comment_char,
+        #due to possible leading replace
+        if replace is None:
+            if separator == " ":
+                if comment_char == "\t":
+                    replace = "_"
+                else:
+                    replace = "\t"
+            else:
+                if comment_char == " ":
+                    replace = "_"
+                else:
+                    replace = " "
+        return map( lambda x: x.replace( separator, replace ), fields )
+
+    def get_default_history_by_trans( self, trans, create=False ):
+        def _create_data_manager_history( user ):
+            history = trans.app.model.History( name='Data Manager History (automatically created)', user=user )
+            data_manager_association = trans.app.model.DataManagerHistoryAssociation( user=user, history=history )
+            trans.sa_session.add_all( ( history, data_manager_association ) )
+            trans.sa_session.flush()
+            return history
+        user = trans.user
+        assert user, 'You must be logged in to use this tool.'
+        history = user.data_manager_histories
+        if not history:
+            #create
+            if create:
+                history = _create_data_manager_history( user )
+            else:
+                history = None
+        else:
+            for history in reversed( history ):
+                history = history.history
+                if not history.deleted:
+                    break
+            if history.deleted:
+                if create:
+                    history = _create_data_manager_history( user )
+                else:
+                    history = None
+        return history
+
+
 # Populate tool_type to ToolClass mappings
 tool_types = {}
-for tool_class in [ Tool, DataDestinationTool, SetMetadataTool, DataSourceTool, AsyncDataSourceTool ]:
+for tool_class in [ Tool, DataDestinationTool, SetMetadataTool, DataSourceTool, AsyncDataSourceTool, DataManagerTool ]:
     tool_types[ tool_class.tool_type ] = tool_class
 
 # ---- Utility classes to be factored out -----------------------------------
     """
     def __nonzero__( self ):
         return bool( self.value )
+    def get_display_text( self, quote=True ):
+        print 'self.input',self.input
+        print 'self.input.tool.app', self.input.tool.app
+        print 'self.value', self.value
+        print 'self.input.value_to_display_text( self.value, self.input.tool.app )', self.input.value_to_display_text( self.value, self.input.tool.app )
+        return pipes.quote( self.input.value_to_display_text( self.value, self.input.tool.app ) )
 
 class RawObjectWrapper( ToolParameterValueWrapper ):
     """

File lib/galaxy/tools/actions/__init__.py

         
         # Set history.
         if not history:
-            history = trans.history
+            history = tool.get_default_history_by_trans( trans, create=True ) #trans..history
         
         out_data = odict()
         # Collect any input datasets from the incoming parameters

File lib/galaxy/tools/data/__init__.py

         return self.data_tables.__getitem__( key )
     def __contains__( self, key ):
         return self.data_tables.__contains__( key )
+    def get( self, name, default=None ):
+        try:
+            return self[ name ]
+        except KeyError:
+            return default
     def load_from_config_file( self, config_filename, tool_data_path, from_shed_config=False ):
         """
         This method is called under 3 conditions:
     def __init__( self, config_element, tool_data_path ):
         self.name = config_element.get( 'name' )
         self.comment_char = config_element.get( 'comment_char' )
+        self.empty_field_value = config_element.get( 'empty_field_value', '' )
+        self.empty_field_values = {}
         for file_elem in config_element.findall( 'file' ):
             # There should only be one file_elem.
             if 'path' in file_elem.attrib:
                 self.tool_data_file = None
         self.tool_data_path = tool_data_path
         self.missing_index_file = None
+    def get_empty_field_by_name( self, name ):
+        return self.empty_field_values.get( name, self.empty_field_value )
     
 class TabularToolDataTable( ToolDataTable ):
     """
             if os.path.exists( filename ):
                 found = True
                 all_rows.extend( self.parse_file_fields( open( filename ) ) )
+                self.filename = filename
             else:
                 # Since the path attribute can include a hard-coded path to a specific directory
                 # (e.g., <file path="tool-data/cg_crr_files.loc" />) which may not be the same value
                     if os.path.exists( corrected_filename ):
                         found = True
                         all_rows.extend( self.parse_file_fields( open( corrected_filename ) ) )
+                        self.filename = corrected_filename
             if not found:
                 self.missing_index_file = filename
                 log.warn( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) )
                 self.columns[name] = index
                 if index > self.largest_index:
                     self.largest_index = index
+                empty_field_value = column_elem.get( 'empty_field_value', None )
+                if empty_field_value is not None:
+                    self.empty_field_values[ name ] = empty_field_value
         assert 'value' in self.columns, "Required 'value' column missing from column def"
         if 'name' not in self.columns:
             self.columns['name'] = self.columns['value']
                 fields = line.split( self.separator )
                 if self.largest_index < len( fields ):
                     rval.append( fields )
-        return rval        
+        return rval
+    def get_column_name_list( self ):
+        rval = []
+        for i in range( self.largest_index + 1 ):
+            found_column = False
+            for name, index in self.columns.iteritems():
+                if index == i:
+                    rval.append( name )
+                    found_column = True
+                    break
+            if not found_column:
+                rval.append( None )
+        return rval
 
 # Registry of tool data types by type_key
 tool_data_table_types = dict( [ ( cls.type_key, cls ) for cls in [ TabularToolDataTable ] ] )

File lib/galaxy/tools/parameters/basic.py

     >>> print p.filter_value( "hg17" )
     hg17
     """
+    def __init__( self, *args, **kwds ):
+        super( GenomeBuildParameter, self ).__init__( *args, **kwds )
+        self.static_options = [ ( value, key, False ) for key, value in util.dbnames ]
     def get_options( self, trans, other_values ):
         if not trans.history:
             yield 'unspecified', '?', False

File lib/galaxy/util/__init__.py

         return curdir
     return join( *rel_list )
 
+def relativize_symlinks( path, start=None, followlinks=False):
+    for root, dirs, files in os.walk( path, followlinks=followlinks ):
+        rel_start = None
+        for file_name in files:
+            symlink_file_name = os.path.join( root, file_name )
+            if os.path.islink( symlink_file_name ):
+                symlink_target = os.readlink( symlink_file_name )
+                if rel_start is None:
+                    if start is None:
+                        rel_start = root
+                    else:
+                        rel_start = start
+                rel_path = relpath( symlink_target, rel_start )
+                os.remove( symlink_file_name )
+                os.symlink( rel_path, symlink_file_name )
+
 def stringify_dictionary_keys( in_dict ):
     #returns a new dictionary
     #changes unicode keys into strings, only works on top level (does not recurse)

File lib/galaxy/webapps/community/config.py

         self.job_handlers = []
         self.tool_handlers = []
         self.tool_runners = []
+        # Error logging with sentry
+        self.sentry_dsn = kwargs.get( 'sentry_dsn', None )
         # Where the tool shed hgweb.config file is stored - the default is the Galaxy installation directory.
         self.hgweb_config_dir = resolve_path( kwargs.get( 'hgweb_config_dir', '' ), self.root )
         # Proxy features

File lib/galaxy/webapps/galaxy/controllers/tool_runner.py

             tool.input_translator.translate( params )
         # We may be visiting Galaxy for the first time ( e.g., sending data from UCSC ),
         # so make sure to create a new history if we've never had one before.
-        history = trans.get_history( create=True )
+        #history = trans.get_history( create=True )
+        history = tool.get_default_history_by_trans( trans, create=True )
         template, vars = tool.handle_input( trans, params.__dict__ )
         if len( params ) > 0:
             trans.log_event( "Tool params: %s" % ( str( params ) ), tool_id=tool_id )

File templates/webapps/galaxy/admin/index.mako

                         <div class="toolTitle"><a href="${h.url_for( controller='admin', action='quotas' )}" target="galaxy_main">Manage quotas</a></div>
                         <div class="toolTitle"><a href="${h.url_for( controller='library_admin', action='browse_libraries' )}" target="galaxy_main">Manage data libraries</a></div>
                         %if trans.app.config.enable_beta_job_managers:
-                            <div class="toolTitle"><a href="${h.url_for( controller='data_admin', action='manage_data' )}" target="galaxy_main">Manage local data</a></div>
+                            <div class="toolTitle"><a href="${h.url_for( controller='data_admin', action='manage_data' )}" target="galaxy_main">Manage old local data</a></div>
                         %endif
+                        ##how to name this?
+                        <div class="toolTitle"><a href="${h.url_for( controller='data_manager' )}" target="galaxy_main">Manage local (cached) data (beta)</a></div>
                     </div>
                 </div>
                 <div class="toolSectionPad"></div>

File tools/data_source/upload.xml

     </param>
     <param name="async_datasets" type="hidden" value="None"/>
     <upload_dataset name="files" title="Specify Files for Dataset" file_type_name="file_type" metadata_ref="files_metadata">
-        <param name="file_data" type="file" size="30" label="File" ajax-upload="true" help="TIP: Due to browser limitations, uploading files larger than 2GB is guaranteed to fail.  To upload large files, use the URL method (below) or FTP (if enabled by the site administrator).">
+        <param name="file_data" type="file" size="30" label="File" ajax-upload="False" help="TIP: Due to browser limitations, uploading files larger than 2GB is guaranteed to fail.  To upload large files, use the URL method (below) or FTP (if enabled by the site administrator).">
         <validator type="expression" message="You will need to reselect the file you specified (%s)." substitute_value_in_message="True">not ( ( isinstance( value, unicode ) or isinstance( value, str ) ) and value != "" )</validator> <!-- use validator to post message to user about needing to reselect the file, since most browsers won't accept the value attribute for file inputs -->
       </param>
       <param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/>