Commits

Init Hello  committed f14ca41

More tool shed functional test enhancements.

  • Participants
  • Parent commits 33d4ab5

Comments (0)

Files changed (13)

File lib/galaxy/visualization/genomes.py

-import os, re, sys, glob
+import os, re, sys, glob, logging
 from bx.seq.twobit import TwoBitFile
 from galaxy.util.json import from_json_string
 from galaxy import model
 from galaxy.util.bunch import Bunch
 
+log = logging.getLogger( __name__ )
+
 # FIXME: copied from tracks.py
 # Message strings returned to browser
 messages = Bunch(
             self.genomes[ key ] = Genome( key, len_file=f )
                 
         # Add genome data (twobit files) to genomes.
-        for line in open( os.path.join( app.config.tool_data_path, "twobit.loc" ) ):
-            if line.startswith("#"): continue
-            val = line.split()
-            if len( val ) == 2:
-                key, path = val
-                if key in self.genomes:
-                    self.genomes[ key ].twobit_file = path
+        # FIXME: If a galaxy instance does not have ~/tool-data/twobit.loc file, the following error is thrown:
+        # IOError: [Errno 2] No such file or directory: '~/tool-data/twobit.loc'
+        try:
+            for line in open( os.path.join( app.config.tool_data_path, "twobit.loc" ) ):
+                if line.startswith("#"): continue
+                val = line.split()
+                if len( val ) == 2:
+                    key, path = val
+                    if key in self.genomes:
+                        self.genomes[ key ].twobit_file = path
+        except IOError, e:
+            log.exception( str( e ) )
                     
     def get_build( self, dbkey ):
         """ Returns build for the given key. """

File test/tool_shed/base/twilltestcase.py

 import galaxy.webapps.community.util.hgweb_config
 import galaxy.model as galaxy_model
-import common, string, os, re, test_db_util
+import common, string, os, re, test_db_util, simplejson
+import galaxy.util as util
 from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib
 from galaxy.tool_shed.encoding_util import tool_shed_encode
 
         self.galaxy_host = os.environ.get( 'GALAXY_TEST_HOST' )
         self.galaxy_port = os.environ.get( 'GALAXY_TEST_PORT' )
         self.galaxy_url = "http://%s:%s" % ( self.galaxy_host, self.galaxy_port )
+        self.shed_tool_data_table_conf = os.environ.get( 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF' )
         self.file_dir = os.environ.get( 'TOOL_SHED_TEST_FILE_DIR', None )
         self.tool_shed_test_file = None
+        self.tool_data_path = os.environ.get( 'GALAXY_TEST_TOOL_DATA_PATH' )
         self.shed_tools_dict = {}
         self.home()
     def browse_category( self, category, strings_displayed=[], strings_not_displayed=[] ):
         repo = hg.repository( ui.ui(), self.get_repo_path( repository ) )
         tip_ctx = repo.changectx( repo.changelog.tip() )
         return tip_ctx.rev() < 0
+    def reset_installed_repository_metadata( self, repository ):
+        url = '/admin_toolshed/reset_repository_metadata?id=%s' % self.security.encode_id( repository.id )
+        self.visit_galaxy_url( url )
+        self.check_for_strings( [ 'Metadata has been reset' ] )
     def reset_repository_metadata( self, repository ):
         url = '/repository/reset_all_metadata?id=%s' % self.security.encode_id( repository.id )
         self.visit_url( url )
+        self.check_for_strings( [ 'All repository metadata has been reset.' ] )
     def revoke_write_access( self, repository, username ):
         url = '/repository/manage_repository?user_access_button=Remove&id=%s&remove_auth=%s' % \
             ( self.security.encode_id( repository.id ), username )
     def tip_has_metadata( self, repository ):
         tip = self.get_repository_tip( repository )
         return test_db_util.get_repository_metadata_by_repository_id_changeset_revision( repository.id, tip )
+    def update_installed_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ):
+        url = '/admin_toolshed/check_for_updates?id=%s' % self.security.encode_id( installed_repository.id )
+        self.visit_galaxy_url( url )
+        self.check_for_strings( strings_displayed, strings_not_displayed )
     def upload_file( self, 
                      repository, 
                      filename, 
         tc.formfile( "1", "file_data", self.get_filename( filename, filepath ) )
         tc.submit( "upload_button" )
         self.check_for_strings( strings_displayed, strings_not_displayed )
+    def verify_installed_repository_metadata_unchanged( self, name, owner ):
+        installed_repository = test_db_util.get_installed_repository_by_name_owner( name, owner )
+        differs = False
+        metadata = installed_repository.metadata
+        self.reset_installed_repository_metadata( installed_repository )
+        new_metadata = installed_repository.metadata
+        # This test assumes that the different metadata components will always appear in the same order. If this is ever not
+        # the case, this test must be updated.
+        for metadata_key in [ 'datatypes', 'tools', 'tool_dependencies', 'repository_dependencies', 'workflows' ]:
+            if ( metadata_key in metadata and metadata_key not in new_metadata ) or \
+               ( metadata_key not in metadata and metadata_key in new_metadata ):
+                differs = True
+                break
+            elif metadata_key not in metadata and metadata_key not in new_metadata:
+                continue
+            else:
+                if metadata[ metadata_key ] != new_metadata[ metadata_key ]:
+                    differs = True
+                    break
+        if differs:
+            raise AssertionError( 'Metadata for installed repository %s differs after metadata reset.' % name )
     def verify_installed_repository_on_browse_page( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ):
         url = '/admin_toolshed/browse_repositories'
         self.visit_galaxy_url( url )
                                     installed_repository.tool_shed, 
                                     installed_repository.installed_changeset_revision ] )
         self.check_for_strings( strings_displayed, strings_not_displayed )
+    def verify_installed_repository_data_table_entries( self, data_tables=[] ):
+        data_table = util.parse_xml( self.shed_tool_data_table_conf )
+        found = False
+        for table_elem in data_table.findall( 'table' ):
+            for data_table in data_tables:
+                if 'name' in table_elem.attrib and table_elem.attrib[ 'name' ] == data_table:
+                    file_elem = table_elem.find( 'file' )
+                    assert os.path.exists( file_elem.attrib[ 'path' ] ), 'Tool data table file %s not found.' % file_elem.path
+                    found = True
+                    break
+                assert found, 'No entry for %s in %s.' % ( data_table, self.shed_tool_data_table_conf )
     def verify_tool_metadata_for_installed_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ):
         repository_id = self.security.encode_id( installed_repository.id )
         for tool in installed_repository.metadata[ 'tools' ]:
             url = '/admin_toolshed/view_tool_metadata?repository_id=%s&tool_id=%s' % ( repository_id, urllib.quote_plus( tool[ 'id' ] ) )
             self.visit_galaxy_url( url )
             self.check_for_strings( strings, strings_not_displayed )
+    def verify_unchanged_repository_metadata( self, repository ):
+        self.check_repository_changelog( repository )
+        html = self.last_page()
+        self.reset_repository_metadata( repository )
+        self.check_repository_changelog( repository, strings_displayed=[ html ] )
     def visit_galaxy_url( self, url ):
         url = '%s%s' % ( self.galaxy_url, url )
         self.visit_url( url )

File test/tool_shed/functional/test_0000_basic_repository_features.py

         tip_changeset = self.get_repository_tip( repository )
         search_fields = dict( tool_id='Filter1', tool_name='filter', tool_version='2.2.0' )
         self.search_for_valid_tools( search_fields=search_fields, strings_displayed=[ tip_changeset ], strings_not_displayed=[] )
+    def test_0085_verify_repository_metadata( self ):
+        '''Verify that resetting the metadata does not change it.'''
+        repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name )
+        self.verify_unchanged_repository_metadata( repository )

File test/tool_shed/functional/test_0020_basic_repository_dependencies.py

         changeset_revision = self.get_repository_tip( datatypes_repository )
         strings_displayed = [ datatypes_repository_name, common.test_user_1_name, changeset_revision, 'Repository dependencies' ]
         self.display_manage_repository_page( repository, strings_displayed=strings_displayed )
+    def test_0040_verify_repository_metadata( self ):
+        '''Verify that resetting the metadata does not change it.'''
+        emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name )
+        datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name )
+        self.verify_unchanged_repository_metadata( emboss_repository )
+        self.verify_unchanged_repository_metadata( datatypes_repository )

File test/tool_shed/functional/test_0030_repository_dependency_revisions.py

             self.display_manage_repository_page( repository, 
                                                  changeset_revision=changeset_revision, 
                                                  strings_displayed=[ str( metadata ) for metadata in repository_dependency_metadata ] )
+    def test_0040_verify_repository_metadata( self ):
+        '''Verify that resetting the metadata does not change it.'''
+        emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name )
+        emboss_5_repository = test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name )
+        emboss_6_repository = test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name )
+        datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name )
+        for repository in [ emboss_repository, emboss_5_repository, emboss_6_repository, datatypes_repository ]:
+            self.verify_unchanged_repository_metadata( repository )

File test/tool_shed/functional/test_0040_repository_circular_dependencies.py

         # In this case, the displayed dependency will specify the tip revision, but this will not always be the case.
         self.check_repository_dependency( filtering_repository, freebayes_repository, self.get_repository_tip( freebayes_repository ) )
         self.check_repository_dependency( freebayes_repository, filtering_repository, self.get_repository_tip( filtering_repository ) )
+    def test_0035_verify_repository_metadata( self ):
+        '''Verify that resetting the metadata does not change it.'''
+        freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name )
+        filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name )
+        for repository in [ freebayes_repository, filtering_repository ]:
+            self.verify_unchanged_repository_metadata( repository )

File test/tool_shed/functional/test_0050_circular_n_levels.py

         for changeset_revision in self.get_repository_metadata_revisions( emboss_repository ):
             self.check_repository_dependency( freebayes_repository, emboss_repository, changeset_revision )
         self.display_manage_repository_page( freebayes_repository, strings_displayed=[ 'Freebayes depends on the filtering repository.' ] )
+    def test_0035_verify_repository_metadata( self ):
+        '''Verify that resetting the metadata does not change it.'''
+        emboss_datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name )
+        emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name )
+        filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name )
+        freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name )
+        for repository in [ emboss_datatypes_repository, emboss_repository, freebayes_repository, filtering_repository ]:
+            self.verify_unchanged_repository_metadata( repository )

File test/tool_shed/functional/test_0060_workflows.py

                           commit_message="Uploaded filtering 2.2.0", 
                           remove_repo_files_not_in_tar='No' )
         self.load_workflow_image( repository, workflow_name, strings_not_displayed=[ '#EBBCB2' ] )
+    def test_0025_verify_repository_metadata( self ):
+        '''Verify that resetting the metadata does not change it.'''
+        repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name )
+        self.verify_unchanged_repository_metadata( repository )

File test/tool_shed/functional/test_1000_install_basic_repository.py

         self.verify_installed_repository_on_browse_page( installed_repository )
         self.display_installed_repository_manage_page( installed_repository, 
                                                        strings_displayed=[ 'Installed tool shed repository', 'Tools', 'Filter1' ] )
-        self.verify_tool_metadata_for_installed_repository( installed_repository )
+        self.verify_tool_metadata_for_installed_repository( installed_repository )
+    def test_0025_verify_installed_repository_metadata( self ):
+        '''Verify that resetting the metadata on an installed repository does not change the metadata.'''
+        self.verify_installed_repository_metadata_unchanged( 'filtering_0000', common.test_user_1_name )

File test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py

                                                        strings_displayed=[ 'Installed tool shed repository', 'Tools', 'FreeBayes' ] )
         self.check_installed_repository_tool_dependencies( installed_repository, dependencies_installed=False )
         self.verify_tool_metadata_for_installed_repository( installed_repository )
+    def test_0020_verify_installed_repository_metadata( self ):
+        '''Verify that resetting the metadata on an installed repository does not change the metadata.'''
+        self.verify_installed_repository_metadata_unchanged( 'freebayes_0010', common.test_user_1_name )
+    def test_0025_verify_sample_files( self ):
+        '''Verify that the installed repository populated shed_tool_data_table.xml and the sample files.'''
+        self.verify_installed_repository_data_table_entries( data_tables=[ 'sam_fa_indexes' ] )
+        

File test/tool_shed/functional/test_1020_install_repository_with_repository_dependencies.py

                                                        strings_displayed=[ 'Installed tool shed repository', 'Tools', 'antigenic' ] )
         self.check_installed_repository_tool_dependencies( installed_repository, dependencies_installed=False )
         self.verify_tool_metadata_for_installed_repository( installed_repository )
+    def test_0020_verify_installed_repository_metadata( self ):
+        '''Verify that resetting the metadata on an installed repository does not change the metadata.'''
+        self.verify_installed_repository_metadata_unchanged( 'emboss_0020', common.test_user_1_name )

File test/tool_shed/functional/test_1030_install_repository_with_dependency_revisions.py

                                                        strings_displayed=[ 'Installed tool shed repository', 'Tools', 'antigenic' ] )
         self.check_installed_repository_tool_dependencies( installed_repository, dependencies_installed=False )
         self.verify_tool_metadata_for_installed_repository( installed_repository )
+        self.update_installed_repository( installed_repository, strings_displayed=[ "there are no updates available" ] )
+    def test_0025_verify_installed_repository_metadata( self ):
+        '''Verify that resetting the metadata on an installed repository does not change the metadata.'''
+        self.verify_installed_repository_metadata_unchanged( 'emboss_0030', common.test_user_1_name )

File test/tool_shed/functional_tests.py

 </toolbox>
 '''
 
+tool_conf_xml = '''<?xml version="1.0"?>
+<toolbox>
+    <section name="Get Data" id="getext">
+        <tool file="data_source/upload.xml"/>
+    </section>
+</toolbox>
+'''
+
 tool_data_table_conf_xml_template = '''<?xml version="1.0"?>
 <tables>
 </tables>
         shed_db_path = os.path.join( tempdir, 'database' )
     shed_tool_data_table_conf_file = os.environ.get( 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF', os.path.join( tool_shed_test_tmp_dir, 'shed_tool_data_table_conf.xml' ) )
     galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_DATA_TABLE_CONF', os.path.join( tool_shed_test_tmp_dir, 'tool_data_table_conf.xml' ) )
-    galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_tool_conf.xml' ) )
+    galaxy_tool_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_tool_conf.xml' ) )
+    galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_TEST_SHED_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_shed_tool_conf.xml' ) )
     galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_TEST_SHED_TOOLS_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_sheds_conf.xml' ) )
+    if 'GALAXY_TEST_TOOL_DATA_PATH' in os.environ:
+        tool_data_path = os.environ.get( 'GALAXY_TEST_TOOL_DATA_PATH' )
+    else:
+        tool_data_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
+        os.environ[ 'GALAXY_TEST_TOOL_DATA_PATH' ] = tool_data_path
     if 'GALAXY_TEST_DBPATH' in os.environ:
         galaxy_db_path = os.environ[ 'GALAXY_TEST_DBPATH' ]
     else: 
     file( galaxy_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template )
     # Generate the shed_tool_data_table_conf.xml file.
     file( shed_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template )
-
+    os.environ[ 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file
     # ---- Build Tool Shed Application -------------------------------------------------- 
     toolshedapp = None 
     global_conf = { '__file__' : 'community_wsgi.ini.sample' }
     
     # ---- Optionally start up a Galaxy instance ------------------------------------------------------
     if 'TOOL_SHED_TEST_OMIT_GALAXY' not in os.environ:
+        # Generate the tool_conf.xml file.
+        file( galaxy_tool_conf_file, 'w' ).write( tool_conf_xml )
         # Generate the shed_tool_conf.xml file.
         tool_sheds_conf_template_parser = string.Template( tool_sheds_conf_xml_template )
         tool_sheds_conf_xml = tool_sheds_conf_template_parser.safe_substitute( shed_url=tool_shed_test_host, shed_port=tool_shed_test_port )
                                                database_engine_option_pool_size = '10',
                                                file_path = galaxy_file_path,
                                                tool_path = tool_path,
+                                               tool_data_path = tool_data_path,
                                                tool_dependency_dir=galaxy_tool_dependency_dir,
                                                shed_tool_path=galaxy_shed_tool_path,
                                                update_integrated_tool_panel = False,
-                                               tool_config_file = galaxy_shed_tool_conf_file,
+                                               tool_config_file = [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ],
                                                tool_sheds_config_file = galaxy_tool_sheds_conf_file,
                                                datatype_converters_config_file = "datatype_converters_conf.xml.sample",
                                                tool_parse_help = False,