Commits

Greg Von Kuster committed ad227f6

Enhance Galaxy tool initialization to do the following:
1. Support multiple tool_conf.xml files with tool_conf.xml being the default. Additional files must use the tool shed XML rules.
2. Allow new added tools to be loaded into a selected tool panel section wothout requiring a Galaxy server restart.
3. Add a new tool shed registry, enabling configuring of tool sheds for communication with the local Galaxy instance.

Also add the version of the tool to the form heading when loaded into tool_fom.mako, and clean up the code for reloading a tool in the tool panel.

Comments (0)

Files changed (9)

lib/galaxy/app.py

 from galaxy import config, jobs, util, tools, web
 import galaxy.tools.search
 import galaxy.tools.data
+import galaxy.tools.tool_shed_registry
 from galaxy.web import security
 import galaxy.model
 import galaxy.datatypes.registry
         # Set up datatypes registry
         self.datatypes_registry = galaxy.datatypes.registry.Registry( self.config.root, self.config.datatypes_config )
         galaxy.model.set_datatypes_registry( self.datatypes_registry )
+        # Set up the tool sheds registry
+        if os.path.isfile( self.config.tool_sheds_config ):
+            self.tool_shed_registry = galaxy.tools.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config )
+        else:
+            self.tool_shed_registry = None
         # Determine the database url
         if self.config.database_connection:
             db_url = self.config.database_connection
         # Tool data tables
         self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_table_config_path )
         # Initialize the tools
-        self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, self )
+        self.toolbox = tools.ToolBox( self.config.tool_configs, self.config.tool_path, self )
         # Search support for tools
         self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox )
         # Load datatype converters

lib/galaxy/config.py

 import sys, os, tempfile
 import logging, logging.config
 import ConfigParser
-from galaxy.util import string_as_bool
+from galaxy.util import string_as_bool, listify
 
 from galaxy import eggs
 import pkg_resources
         self.enable_api = string_as_bool( kwargs.get( 'enable_api', False ) )
         self.enable_openid = string_as_bool( kwargs.get( 'enable_openid', False ) )
         self.enable_quotas = string_as_bool( kwargs.get( 'enable_quotas', False ) )
+        self.tool_sheds_config = kwargs.get( 'tool_sheds_config_file', 'tool_sheds_conf.xml' )
         self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
         self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() )
         self.len_file_path = kwargs.get( "len_file_path", resolve_path(os.path.join(self.tool_data_path, 'shared','ucsc','chrom'), self.root) )
         self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
-        self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root )        
+        self.tool_configs = [ resolve_path( p, self.root ) for p in listify( kwargs.get( 'tool_config_file', 'tool_conf.xml' ) ) ]    
         self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root )
         self.tool_secret = kwargs.get( "tool_secret", "" )
         self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
                 except Exception, e:
                     raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) )
         # Check that required files exist
-        for path in self.tool_config, self.datatypes_config:
+        for path in self.tool_configs:
             if not os.path.isfile(path):
                 raise ConfigurationError("File not found: %s" % path )
+        if not os.path.isfile( self.datatypes_config ):
+            raise ConfigurationError("File not found: %s" % path )
         # Check for deprecated options.
         for key in self.config_dict.keys():
             if key in self.deprecated_options:

lib/galaxy/tools/__init__.py

 from galaxy.datatypes import sniff
 from cgi import FieldStorage
 from galaxy.util.hash_util import *
+from galaxy.util import listify
 
 log = logging.getLogger( __name__ )
 
     Container for a collection of tools
     """
 
-    def __init__( self, config_filename, tool_root_dir, app ):
+    def __init__( self, config_filenames, tool_root_dir, app ):
         """
         Create a toolbox from the config file names by `config_filename`,
         using `tool_root_directory` as the base directory for finding 
         individual tool config files.
         """
+        # The shed_tool_confs dictionary contains shed_conf_filename : tool_path pairs.
+        self.shed_tool_confs = {}
         self.tools_by_id = {}
         self.workflows_by_id = {}
         self.tool_panel = odict()
+        # The following refers to the tool_path config setting for backward compatibility.
+        # Additional newer (e.g., shed_tool_conf.xml) files include the tool_path attribute
+        # within the <toolbox> tag.
         self.tool_root_dir = tool_root_dir
         self.app = app
         self.init_dependency_manager()
-        try:
-            self.init_tools( config_filename )
-        except:
-            log.exception( "ToolBox error reading %s", config_filename )
-
+        for config_filename in listify( config_filenames ):
+            try:
+                self.init_tools( config_filename )
+            except:
+                log.exception( "ToolBox error reading %s", config_filename )
     def init_tools( self, config_filename ):
         """
         Read the configuration file and load each tool.
             </section>
         </toolbox>
         """
-        def load_tool( elem, panel_dict ):
-            try:
-                path = elem.get( "file" )
-                tool = self.load_tool( os.path.join( self.tool_root_dir, path ) )
-                if self.app.config.get_bool( 'enable_tool_tags', False ):
-                    tag_names = elem.get( "tags", "" ).split( "," )
-                    for tag_name in tag_names:
-                        if tag_name == '':
-                            continue
-                        tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first()
-                        if not tag:
-                            tag = self.app.model.Tag( name=tag_name )
-                            self.sa_session.add( tag )
-                            self.sa_session.flush()
-                            tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
-                            self.sa_session.add( tta )
-                            self.sa_session.flush()
-                        else:
-                            for tagged_tool in tag.tagged_tools:
-                                if tagged_tool.tool_id == tool.id:
-                                    break
-                            else:
-                                tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
-                                self.sa_session.add( tta )
-                                self.sa_session.flush()
-                self.tools_by_id[ tool.id ] = tool
-                key = 'tool_' + tool.id
-                panel_dict[ key ] = tool
-                log.debug( "Loaded tool: %s %s" % ( tool.id, tool.version ) )
-            except:
-                log.exception( "error reading tool from path: %s" % path )
-        def load_workflow( elem, panel_dict ):
-            try:
-                # TODO: should id be encoded?
-                workflow_id = elem.get( 'id' )
-                workflow = self.load_workflow( workflow_id )
-                self.workflows_by_id[ workflow_id ] = workflow
-                key = 'workflow_' + workflow_id
-                panel_dict[ key ] = workflow
-                log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) )
-            except:
-                log.exception( "error loading workflow: %s" % workflow_id )
-        def load_label( elem, panel_dict ):
-            label = ToolSectionLabel( elem )
-            key = 'label_' + label.id
-            panel_dict[ key ] = label
-        def load_section( elem, panel_dict ):
-            section = ToolSection( elem )
-            log.debug( "Loading section: %s" % section.name )
-            for section_elem in elem:
-                if section_elem.tag == 'tool':
-                    load_tool( section_elem, section.elems )
-                elif section_elem.tag == 'workflow':
-                    load_workflow( section_elem, section.elems )
-                elif section_elem.tag == 'label':
-                    load_label( section_elem, section.elems )
-            key = 'section_' + section.id
-            panel_dict[ key ] = section
-                
         if self.app.config.get_bool( 'enable_tool_tags', False ):
             log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")")
             self.sa_session.query( self.app.model.ToolTagAssociation ).delete()
             self.sa_session.flush()
-        log.info("parsing the tool configuration")
+        log.info( "parsing the tool configuration %s" % config_filename )
         tree = util.parse_xml( config_filename )
         root = tree.getroot()
+        tool_path = root.get( 'tool_path' )
+        if tool_path:
+            # We're parsing a shed_tool_conf file since we have a tool_path attribute.
+            self.shed_tool_confs[ config_filename ] = tool_path
+        else:
+            # Default to backward compatible config setting.
+            tool_path = self.tool_root_dir
         for elem in root:
             if elem.tag == 'tool':
-                load_tool( elem, self.tool_panel )
+                self.load_tool_tag_set( elem, self.tool_panel, tool_path, guid=elem.get( 'guid' ) )
             elif elem.tag == 'workflow':
-                load_workflow( elem, self.tool_panel )
+                self.load_workflow_tag_set( elem, self.tool_panel )
             elif elem.tag == 'section' :
-                load_section( elem, self.tool_panel )
+                self.load_section_tag_set( elem, self.tool_panel, tool_path )
             elif elem.tag == 'label':
-                load_label( elem, self.tool_panel )
-        
-    def load_tool( self, config_file ):
+                self.load_label_tag_set( elem, self.tool_panel )
+    def load_tool_tag_set( self, elem, panel_dict, tool_path, guid=None ):
+        try:
+            path = elem.get( "file" )
+            tool = self.load_tool( os.path.join( tool_path, path ), guid=guid )
+            if self.app.config.get_bool( 'enable_tool_tags', False ):
+                tag_names = elem.get( "tags", "" ).split( "," )
+                for tag_name in tag_names:
+                    if tag_name == '':
+                        continue
+                    tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first()
+                    if not tag:
+                        tag = self.app.model.Tag( name=tag_name )
+                        self.sa_session.add( tag )
+                        self.sa_session.flush()
+                        tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
+                        self.sa_session.add( tta )
+                        self.sa_session.flush()
+                    else:
+                        for tagged_tool in tag.tagged_tools:
+                            if tagged_tool.tool_id == tool.id:
+                                break
+                        else:
+                            tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
+                            self.sa_session.add( tta )
+                            self.sa_session.flush()
+            if tool.id in self.tools_by_id:
+                raise Exception( "Tool with id %s already loaded." % tool.id )
+            else:
+                self.tools_by_id[ tool.id ] = tool
+            key = 'tool_' + tool.id
+            panel_dict[ key ] = tool
+            log.debug( "Loaded tool: %s %s" % ( tool.id, tool.version ) )
+        except:
+            log.exception( "error reading tool from path: %s" % path )
+    def load_workflow_tag_set( self, elem, panel_dict ):
+        try:
+            # TODO: should id be encoded?
+            workflow_id = elem.get( 'id' )
+            workflow = self.load_workflow( workflow_id )
+            self.workflows_by_id[ workflow_id ] = workflow
+            key = 'workflow_' + workflow_id
+            panel_dict[ key ] = workflow
+            log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) )
+        except:
+            log.exception( "error loading workflow: %s" % workflow_id )
+    def load_label_tag_set( self, elem, panel_dict ):
+        label = ToolSectionLabel( elem )
+        key = 'label_' + label.id
+        panel_dict[ key ] = label
+    def load_section_tag_set( self, elem, panel_dict, tool_path ):
+        key = 'section_' + elem.get( "id" )
+        if key in panel_dict:
+            # Appending a tool to an existing section in self.tool_panel
+            elems = panel_dict[ key ].elems
+            log.debug( "Appending to section: %s" % elem.get( "name" ) )
+        else:
+            # Appending a new section to self.tool_panel
+            section = ToolSection( elem )
+            elems = section.elems
+            log.debug( "Loading section: %s" % section.name )
+        for section_elem in elem:
+            if section_elem.tag == 'tool':
+                self.load_tool_tag_set( section_elem, elems, tool_path, guid=section_elem.get( 'guid' ) )
+            elif section_elem.tag == 'workflow':
+                self.load_workflow_tag_set( section_elem, elems )
+            elif section_elem.tag == 'label':
+                self.load_label_tag_set( section_elem, elems )
+        if key not in panel_dict:
+            panel_dict[ key ] = section
+    def load_tool( self, config_file, guid=None ):
         """
         Load a single tool from the file named by `config_file` and return 
         an instance of `Tool`.
             type_elem = root.find( "type" )
             module = type_elem.get( 'module', 'galaxy.tools' )
             cls = type_elem.get( 'class' )
-            mod = __import__( module, globals(), locals(), [cls])
+            mod = __import__( module, globals(), locals(), [cls] )
             ToolClass = getattr( mod, cls )
         elif root.get( 'tool_type', None ) is not None:
             ToolClass = tool_types.get( root.get( 'tool_type' ) )
         else:
             ToolClass = Tool
-        return ToolClass( config_file, root, self.app )
-        
-    def reload( self, tool_id ):
+        return ToolClass( config_file, root, self.app, guid=guid )
+    def reload_tool_by_id( self, tool_id ):
         """
         Attempt to reload the tool identified by 'tool_id', if successful
         replace the old tool.
         """
         if tool_id not in self.tools_by_id:
-            raise ToolNotFoundException( "No tool with id %s" % tool_id )
-        old_tool = self.tools_by_id[ tool_id ]
-        new_tool = self.load_tool( old_tool.config_file )
-        # Replace old_tool with new_tool in self.tool_panel
-        tool_key = 'tool_' + tool_id
-        for key, val in self.tool_panel.items():
-            if key == tool_key:
-                self.tool_panel[ key ] = new_tool
-                break
-            elif key.startswith( 'section' ):
-                section = val
-                for section_key, section_val in section.elems.items():
-                    if section_key == tool_key:
-                        self.tool_panel[ key ].elems[ section_key ] = new_tool
-                        break
-        self.tools_by_id[ tool_id ] = new_tool
-        log.debug( "Reloaded tool %s %s" %( old_tool.id, old_tool.version ) )
-
+            message = "No tool with id %s" % tool_id
+            status = 'error'
+        else:
+            old_tool = self.tools_by_id[ tool_id ]
+            new_tool = self.load_tool( old_tool.config_file )
+            # Replace old_tool with new_tool in self.tool_panel
+            tool_key = 'tool_' + tool_id
+            for key, val in self.tool_panel.items():
+                if key == tool_key:
+                    self.tool_panel[ key ] = new_tool
+                    break
+                elif key.startswith( 'section' ):
+                    section = val
+                    for section_key, section_val in section.elems.items():
+                        if section_key == tool_key:
+                            self.tool_panel[ key ].elems[ section_key ] = new_tool
+                            break
+            self.tools_by_id[ tool_id ] = new_tool
+            message = "Reloaded the tool:<br/>"
+            message += "<b>name:</b> %s<br/>" % old_tool.name
+            message += "<b>id:</b> %s<br/>" % old_tool.id
+            message += "<b>version:</b> %s" % old_tool.version
+            status = 'done'
+        return message, status
     def load_workflow( self, workflow_id ):
         """
         Return an instance of 'Workflow' identified by `id`, 
     
     tool_type = 'default'
     
-    def __init__( self, config_file, root, app ):
+    def __init__( self, config_file, root, app, guid=None ):
         """
         Load a tool from the config named by `config_file`
         """
         self.tool_dir = os.path.dirname( config_file )
         self.app = app
         # Parse XML element containing configuration
-        self.parse( root )
+        self.parse( root, guid=guid )
     
     @property
     def sa_session( self ):
         """
         return self.app.model.context
     
-    def parse( self, root ):
+    def parse( self, root, guid=None ):
         """
         Read tool configuration from the element `root` and fill in `self`.
         """
             raise Exception, "Missing tool 'name'"
         # Get the UNIQUE id for the tool 
         # TODO: can this be generated automatically?
-        self.id = root.get( "id" )
+        if guid is not None:
+            self.id = guid
+        else:
+            self.id = root.get( "id" )
         if not self.id: 
             raise Exception, "Missing tool 'id'" 
         self.version = root.get( "version" )

lib/galaxy/tools/tool_shed_registry.py

+import sys, logging
+from galaxy.util import parse_xml
+from galaxy.util.odict import odict
+
+log = logging.getLogger( __name__ )
+
+if sys.version_info[:2] == ( 2, 4 ):
+    from galaxy import eggs
+    eggs.require( 'ElementTree' )
+    from elementtree import ElementTree
+else:
+    from xml.etree import ElementTree
+
+class Registry( object ):
+    def __init__( self, root_dir=None, config=None ):
+        self.tool_sheds = odict()
+        if root_dir and config:
+            # Parse datatypes_conf.xml
+            tree = parse_xml( config )
+            root = tree.getroot()
+            # Load datatypes and converters from config
+            log.debug( 'Loading references to tool sheds from %s' % config )
+            for elem in root.findall( 'tool_shed' ):
+                try:
+                    name = elem.get( 'name', None )
+                    url = elem.get( 'url', None )
+                    if name and url:
+                        self.tool_sheds[ name ] = url
+                        log.debug( 'Loaded reference to tool shed: %s' % name )
+                except Exception, e:
+                    log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) )

templates/admin/reload_tool.mako

 <div class="toolForm">
     <div class="toolFormTitle">Reload Tool</div>
     <div class="toolFormBody">
-    <form name="tool_reload" action="${h.url_for( controller='admin', action='tool_reload' )}" method="post" >
+    <form name="reload_tool" id="reload_tool" action="${h.url_for( controller='admin', action='reload_tool' )}" method="post" >
         <div class="form-row">
             <label>
                 Tool to reload:
             </select>
         </div>
         <div class="form-row">
-            <input type="submit" name="action" value="Reload"/>
+            <input type="submit" name="reload_tool_button" value="Reload"/>
         </div>
     </form>
     </div>

templates/tool_form.mako

     %if tool.has_multiple_pages:
         <div class="toolFormTitle">${tool.name} (step ${tool_state.page+1} of ${tool.npages})</div>
     %else:
-        <div class="toolFormTitle">${tool.name}</div>
+        <div class="toolFormTitle">${tool.name} (version ${tool.version})</div>
     %endif
     <div class="toolFormBody">
         <form id="tool_form" name="tool_form" action="${tool_url}" enctype="${tool.enctype}" target="${tool.target}" method="${tool.method}">

templates/webapps/community/repository/preview_tools_in_changeset.mako

+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+<%namespace file="/webapps/community/common/common.mako" import="*" />
+<%namespace file="/webapps/community/repository/common.mako" import="*" />
+
+<%
+    is_new = repository.is_new
+    can_contact_owner = trans.user and trans.user != repository.user
+    can_push = trans.app.security_agent.can_push( trans.user, repository )
+    can_rate = not is_new and trans.user and repository.user != trans.user
+    can_upload = can_push
+    can_download = not is_new and ( not is_malicious or can_push )
+    can_browse_contents = not is_new
+    can_view_change_log = not is_new
+    if can_push:
+        browse_label = 'Browse or delete repository files'
+    else:
+        browse_label = 'Browse repository files'
+%>
+
+<%!
+   def inherit(context):
+       if context.get('use_panels'):
+           return '/webapps/community/base_panels.mako'
+       else:
+           return '/base.mako'
+%>
+<%inherit file="${inherit(context)}"/>
+
+<%def name="stylesheets()">
+    ${parent.stylesheets()}
+    ${h.css( "jquery.rating" )}
+    <style type="text/css">
+    ul.fileBrowser,
+    ul.toolFile {
+        margin-left: 0;
+        padding-left: 0;
+        list-style: none;
+    }
+    ul.fileBrowser {
+        margin-left: 20px;
+    }
+    .fileBrowser li,
+    .toolFile li {
+        padding-left: 20px;
+        background-repeat: no-repeat;
+        background-position: 0;
+        min-height: 20px;
+    }
+    .toolFile li {
+        background-image: url( ${h.url_for( '/static/images/silk/page_white_compressed.png' )} );
+    }
+    .fileBrowser li {
+        background-image: url( ${h.url_for( '/static/images/silk/page_white.png' )} );
+    }
+    </style>
+</%def>
+
+<%def name="javascripts()">
+    ${parent.javascripts()}
+    ${h.js( "jquery.rating" )}
+    ${common_javascripts(repository)}
+</%def>
+
+<br/><br/>
+<ul class="manage-table-actions">
+    <a class="action-button" href="${h.url_for( controller='repository', action='install_repository_revision', repository_id=trans.security.encode_id( repository.id ), webapp='community', changeset_revision=changeset_revision, galaxy_url=galaxy_url )}">Install to local Galaxy</a>
+</ul>
+
+%if message:
+    ${render_msg( message, status )}
+%endif
+
+<div class="toolForm">
+    <div class="toolFormTitle">Repository ${repository.name}</div>
+    <div class="toolFormBody">
+        %if len( changeset_revision_select_field.options ) > 1:
+            <form name="change_revision" id="change_revision" action="${h.url_for( controller='repository', action='preview_tools_in_changeset', repository_id=trans.security.encode_id( repository.id ), galaxy_url=galaxy_url )}" method="post" >
+                <div class="form-row">
+                    <%
+                        if changeset_revision == repository.tip:
+                            tip_str = 'repository tip'
+                        else:
+                            tip_str = ''
+                    %>
+                    ${changeset_revision_select_field.get_html()} <i>${tip_str}</i>
+                    <div class="toolParamHelp" style="clear: both;">
+                        Select a revision to inspect and download versions of tools from this repository.
+                    </div>
+                </div>
+            </form>
+        %else:
+            <div class="form-row">
+                <label>Revision:</label>
+                ${revision_label}
+            </div>
+        %endif
+    </div>
+</div>
+<p/>
+${render_repository_tools_and_workflows( metadata, display_for_install=display_for_install, galaxy_url=galaxy_url )}

tool_sheds_conf.xml.sample

+<?xml version="1.0"?>
+<tool_sheds>
+    <tool_shed name="Galaxy main tool shed" url="http://toolshed.g2.bx.psu.edu/"/>
+    <tool_shed name="Galaxy test tool shed" url="http://testtoolshed.g2.bx.psu.edu/"/>
+</tool_sheds>

universe_wsgi.ini.sample

 # Temporary files are stored in this directory.
 #new_file_path = database/tmp
 
-# Tool config file, defines what tools are available in Galaxy.
+# Tool config files, defines what tools are available in Galaxy.
+# Tools can be locally developed or installed from tool sheds.
 #tool_config_file = tool_conf.xml
 
-# Path to the directory containing the tools defined in the config.
+# Default path to the directory containing the tools defined in tool_conf.xml.
+# Other tool config files must include the tool_path as an attribute in the <toolbox> tag.
 #tool_path = tools
 
 # Directory where data used by tools is located, see the samples in that