Commits

Dave B. committed b9da077

Tool dependency definitions: Add <actions_group> tag and support for filtering contained <actions> tags by architecture and operating system.

  • Participants
  • Parent commits 4a4a84d

Comments (0)

Files changed (4)

lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py

                     # Eliminate the download_by_url action so remaining actions can be processed correctly.
                     filtered_actions = actions[ 1: ]
                     url = action_dict[ 'url' ]
+                    is_binary = action_dict.get( 'is_binary', False )
+                    log.debug( 'Attempting to download via url: %s', url )
                     if 'target_filename' in action_dict:
-                        # Sometimes compressed archives extracts their content to a folder other than the default defined file name.  Using this
+                        # Sometimes compressed archives extract their content to a folder other than the default defined file name.  Using this
                         # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction.
                         downloaded_filename = action_dict[ 'target_filename' ]
                     else:
                         downloaded_filename = os.path.split( url )[ -1 ]
                     dir = td_common_util.url_download( work_dir, downloaded_filename, url, extract=True )
+                    if is_binary:
+                        log_file = os.path.join( install_dir, INSTALLATION_LOG )
+                        log.debug( 'log_file: %s' % log_file )
+                        if os.path.exists( log_file ):
+                            logfile = open( log_file, 'ab' )
+                        else:
+                            logfile = open( log_file, 'wb' )
+                        logfile.write( 'Successfully downloaded from url: %s\n' % action_dict[ 'url' ] )
+                        logfile.close()
+                    log.debug( 'Successfully downloaded from url: %s' % action_dict[ 'url' ] )
                 elif action_type == 'shell_command':
                     # <action type="shell_command">git clone --recursive git://github.com/ekg/freebayes.git</action>
                     # Eliminate the shell_command clone action so remaining actions can be processed correctly.

lib/tool_shed/galaxy_install/tool_dependencies/install_util.py

                                                                                              type='package',
                                                                                              status=app.model.ToolDependency.installation_status.INSTALLING,
                                                                                              set_status=True )
+                    # Get the information that defines the current platform.
+                    platform_info_dict = tool_dependency_util.get_platform_info_dict()
                     if package_install_version == '1.0':
-                        # Handle tool dependency installation using a fabric method included in the Galaxy framework.
-                        for actions_elem in package_elem:
-                            install_via_fabric( app, tool_dependency, actions_elem, install_dir, package_name=package_name )
-                            sa_session.refresh( tool_dependency )
-                            if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
-                                print package_name, 'version', package_version, 'installed in', install_dir
+                        # Handle tool dependency installation using a fabric method included in the Galaxy framework.  The first thing we do
+                        # is check the installation architecture to see if we have a precompiled binary that works on the target system.
+                        binary_installed = False
+                        actions_elem_tuples = []
+                        # Build a list of grouped and ungrouped <actions> tagsets to be processed in the order they are defined in the
+                        # tool_dependencies.xml file.
+                        for elem in package_elem:
+                            # Default to not treating actions as grouped.
+                            grouped = False
+                            # Skip any element that is not <actions> or <actions_group>. This will also skip comments and <readme> tags.
+                            if elem.tag not in [ 'actions', 'actions_group' ]:
+                                continue
+                            if elem.tag == 'actions':
+                                # We have an <actions> tag that should not be matched against a specific combination of architecture and operating system.
+                                grouped = False
+                                actions_elem_tuples.append( ( grouped, elem ) )
+                            else:
+                                # Record the number of <actions> elements, in order to filter out any <action> elements that precede <actions>
+                                # elements.
+                                actions_elem_count = len( elem.findall( 'actions' ) )
+                                # Record the number of <actions> elements that have architecture and os specified, in order to filter out any
+                                # platform-independent <actions> elements that come before platform-specific <actions> elements. This call to
+                                # elem.findall is filtered by tags that have both the os and architecture specified.
+                                # For more details, see http://docs.python.org/2/library/xml.etree.elementtree.html Section 19.7.2.1.
+                                platform_actions_element_count = len( elem.findall( 'actions[@architecture][@os]' ) )
+                                platform_actions_elements_processed = 0
+                                actions_elems_processed = 0
+                                # We have an actions_group element, and its child <actions> elements should therefore be compared with the current 
+                                # operating system and processor architecture.
+                                grouped = True
+                                # The tagsets that will go into the actions_elem_list are those that install a precompiled binary if the 
+                                # architecture and operating system match its defined attributes.  If precompiled binary is not installed
+                                # the first <actions> tag following those that have the os and architecture attributes will be processed
+                                # in order to install and compile the source.
+                                actions_elem_list = []
+                                # The tagsets that will go into the after_install_actions list are <action> tags instead of <actions> tags.  These
+                                # will only be processed if they are at the end of the <actions_group> tagset. See below for details.
+                                after_install_actions = []
+                                platform_independent_actions = []
+                                # Loop through the <actions_group> element and build the actions_elem_list and the after_install_actions list.
+                                for child_element in elem:
+                                    if child_element.tag == 'actions':
+                                        actions_elems_processed += 1
+                                        system = child_element.get( 'os' )
+                                        architecture = child_element.get( 'architecture' )
+                                        # Skip <actions> tags that have only one of architecture or os specified, in order for the count in
+                                        # platform_actions_elements_processed to remain accurate.
+                                        if ( system and not architecture ) or ( architecture and not system ):
+                                            log.debug( 'Error: Both architecture and os attributes must be specified in an <actions> tag.' )
+                                            continue
+                                        # Since we are inside an <actions_group> tagset, compare it with our current platform information and filter
+                                        # the <actions> tagsets that don't match. Require both the os and architecture attributes to be defined in
+                                        # order to find a match.
+                                        if system and architecture:
+                                            platform_actions_elements_processed += 1
+                                            # If either the os or architecture do not match the platform, this <actions> tag will not be considered
+                                            # a match. Skip it and proceed with checking the next one.
+                                            if platform_info_dict[ 'os' ] != system or platform_info_dict[ 'architecture' ] != architecture:
+                                                continue
+                                        else:
+                                            # <actions> tags without both os and architecture attributes are only allowed to be specified after
+                                            # platform-specific <actions> tags. If we find a platform-independent <actions> tag before all
+                                            # platform-specific <actions> tags have been processed, log a message stating this and skip to the
+                                            # next <actions> tag.
+                                            if platform_actions_elements_processed < platform_actions_element_count:
+                                                message = 'Error: <actions> tags without os and architecture attributes are only allowed '
+                                                message += 'after <actions> tags with os and architecture attributes specified. Skipping '
+                                                message += 'current <actions> tag.'
+                                                log.debug( message )
+                                                continue
+                                        # If we reach this point, it means one of two things: 1) The system and architecture attributes are not
+                                        # defined in this <actions> tag, or 2) The system and architecture attributes are defined, and they are
+                                        # an exact match for the current platform. Append the child element to the list of elements to process.
+                                        actions_elem_list.append( child_element )
+                                    elif child_element.tag == 'action':
+                                        # Any <action> tags within an <actions_group> tagset must come after all <actions> tags. 
+                                        if actions_elems_processed == actions_elem_count:
+                                            # If all <actions> elements have been processed, then this <action> element can be appended to the
+                                            # list of actions to execute within this group.
+                                            after_install_actions.append( child_element )
+                                        else:
+                                            # If any <actions> elements remain to be processed, then log a message stating that <action>
+                                            # elements are not allowed to precede any <actions> elements within an <actions_group> tagset.
+                                            message = 'Error: <action> tags are only allowed at the end of an <actions_group> '
+                                            message += 'tagset, after all <actions> tags. '
+                                            message += 'Skipping <%s> element with type %s.' % ( child_element.tag, child_element.get( 'type' ) )
+                                            log.debug( message )
+                                            continue
+                                if after_install_actions:
+                                    actions_elem_list.extend( after_install_actions )
+                                actions_elem_tuples.append( ( grouped, actions_elem_list ) )
+                        # At this point we have a list of <actions> elems that are either defined within an <actions_group> tagset, and filtered by
+                        # the current platform, or not defined within an <actions_group> tagset, and not filtered.
+                        for grouped, actions_elems in actions_elem_tuples:
+                            if grouped:
+                                # Platform matching is only performed inside <actions_group> tagsets, os and architecture attributes are otherwise ignored.
+                                for actions_elem in actions_elems:
+                                    system = actions_elem.get( 'os' )
+                                    architecture = actions_elem.get( 'architecture' )
+                                    # If this <actions> element has the os and architecture attributes defined, then we only want to process
+                                    # until a successful installation is achieved.
+                                    if system and architecture:
+                                        # If an <actions> tag has been defined that matches our current platform, and the recipe specified
+                                        # within that <actions> tag has been successfully processed, skip any remaining platform-specific
+                                        # <actions> tags.
+                                        if binary_installed:
+                                            continue
+                                        # No platform-specific <actions> recipe has yet resulted in a successful installation.
+                                        install_via_fabric( app, 
+                                                            tool_dependency, 
+                                                            install_dir, 
+                                                            package_name=package_name, 
+                                                            actions_elem=actions_elem, 
+                                                            action_elem=None )
+                                        sa_session.refresh( tool_dependency )
+                                        if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
+                                            # If an <actions> tag was found that matches the current platform, and the install_via_fabric method 
+                                            # did not result in an error state, set binary_installed to True in order to skip any remaining 
+                                            # platform-specific <actions> tags.
+                                            if not binary_installed:
+                                                binary_installed = True
+                                        else:
+                                            # Otherwise, move on to the next matching <actions> tag, or any defined <actions> tags that do not
+                                            # contain platform-dependent recipes.
+                                            if binary_installed:
+                                                binary_installed = False
+                                            print 'Encountered an error downloading binary for %s version %s: %s' % \
+                                                  ( package_name, package_version, tool_dependency.error_message )
+                                    else:
+                                        # If no <actions> tags have been defined that match our current platform, or none of the matching
+                                        # <actions> tags resulted in a successful tool dependency status, proceed with one and only one
+                                        # <actions> tag that is not defined to be platform-specific.
+                                        if not binary_installed:
+                                            log.debug( 'Platform-specific recipe failed or not found. Proceeding with platform-independent install recipe.' )
+                                            install_via_fabric( app, 
+                                                                tool_dependency, 
+                                                                install_dir, 
+                                                                package_name=package_name, 
+                                                                actions_elem=actions_elem, 
+                                                                action_elem=None )
+                                            break
+                                    # Perform any final actions that have been defined within the actions_group tagset, but outside of 
+                                    # an <actions> tag, such as a set_environment entry, or a download_file or download_by_url command to
+                                    # retrieve extra data for this tool dependency. Only do this if the tool dependency is not in an error
+                                    # state, otherwise skip this action.
+                                    if actions_elem.tag == 'action' and tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
+                                        install_via_fabric( app, 
+                                                            tool_dependency, 
+                                                            install_dir, 
+                                                            package_name=package_name, 
+                                                            actions_elem=None, 
+                                                            action_elem=actions_elem )
+                            else:
+                                # <actions> tags outside of an <actions_group> tag shall not check os or architecture, and if the attributes are
+                                # defined, they will be ignored. All <actions> tags outside of an <actions_group> tagset shall always be processed.
+                                # This is the default and original behavior of the install_package method.
+                                install_via_fabric( app, 
+                                                    tool_dependency, 
+                                                    install_dir, 
+                                                    package_name=package_name, 
+                                                    actions_elem=actions_elems, 
+                                                    action_elem=None )
+                                sa_session.refresh( tool_dependency )
+                                if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR:
+                                    print package_name, 'version', package_version, 'installed in', install_dir
                     else:
                         raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' )
             elif package_elem.tag == 'readme':
             #    print 'Installing tool dependencies via fabric script ', proprietary_fabfile_path
     return tool_dependency
 
-def install_via_fabric( app, tool_dependency, actions_elem, install_dir, package_name=None, proprietary_fabfile_path=None, **kwd ):
+def install_via_fabric( app, tool_dependency, install_dir, package_name=None, proprietary_fabfile_path=None, actions_elem=None, action_elem=None, **kwd ):
     """Parse a tool_dependency.xml file's <actions> tag set to gather information for the installation via fabric."""
     sa_session = app.model.context.current
 
     actions = []
     all_env_shell_file_paths = []
     env_var_dicts = []
-    # Make sure to skip all comments, since they are now included in the XML tree.
-    for action_elem in actions_elem.findall( 'action' ):
+    if actions_elem is not None:
+        elems = actions_elem
+        if elems.get( 'architecture' ) is not None:
+            is_binary_download = True
+        else:
+            is_binary_download = False
+    elif action_elem:
+        # We were provided with a single <action> element to perform certain actions after a platform-specific tarball was downloaded.
+        elems = [ action_elem ]
+    else:
+        elems = []
+    for action_elem in elems:
+        # Make sure to skip all comments, since they are now included in the XML tree.
+        if action_elem.tag != 'action':
+            continue
         action_dict = {}
         action_type = action_elem.get( 'type', 'shell_command' )
         if action_type == 'download_binary':
                 raise Exception( "Unsupported template language '%s' in tool dependency definition." % str( language ) )
         elif action_type == 'download_by_url':
             # <action type="download_by_url">http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2</action>
+            if is_binary_download:
+                action_dict[ 'is_binary' ] = True
             if action_elem.text:
                 action_dict[ 'url' ] = action_elem.text
                 target_filename = action_elem.get( 'target_filename', None )

lib/tool_shed/util/common_install_util.py

                         tool_dependency = install_package( app, elem, tool_shed_repository, tool_dependencies=tool_dependencies )
                     except Exception, e:
                         error_message = "Error installing tool dependency %s version %s: %s" % ( str( package_name ), str( package_version ), str( e ) )
-                        log.debug( error_message )
+                        log.exception( error_message )
                         if tool_dependency:
                             tool_dependency.status = app.model.ToolDependency.installation_status.ERROR
                             tool_dependency.error_message = error_message

lib/tool_shed/util/tool_dependency_util.py

 
 def get_download_url_for_platform( url_templates, platform_info_dict ):
     '''
-    Compare the dict returned by get_platform_info() with the values specified in the base_url element. Return
+    Compare the dict returned by get_platform_info() with the values specified in the url_template element. Return
     true if and only if all defined attributes match the corresponding dict entries. If an entry is not
-    defined in the base_url element, it is assumed to be irrelevant at this stage. For example,
-    <base_url os="darwin">http://hgdownload.cse.ucsc.edu/admin/exe/macOSX.${architecture}/faToTwoBit</base_url>
+    defined in the url_template element, it is assumed to be irrelevant at this stage. For example,
+    <url_template os="darwin">http://hgdownload.cse.ucsc.edu/admin/exe/macOSX.${architecture}/faToTwoBit</url_template>
     where the OS must be 'darwin', but the architecture is filled in later using string.Template.
     '''
     os_ok = False