Commits

James Taylor committed 4d1e1f7

Workflow speed improvements

  • Participants
  • Parent commits 7723618

Comments (0)

Files changed (5)

+diff --git a/eggs.ini b/eggs.ini
+--- a/eggs.ini
++++ b/eggs.ini
+@@ -55,6 +55,9 @@
+ wchartype = 0.1
+ boto = 1.8d
+ 
++repoze.profile = 0.6
++meld3 = 0.6.5
++
+ ; extra version information
+ [tags]
+ psycopg2 = _8.4.2_static
+@@ -72,5 +75,8 @@
+ psycopg2 = postgresql-8.4.2
+ pysqlite = sqlite-amalgamation-3_6_17
+ 
++repoze.profile = http://pypi.python.org/packages/source/r/repoze.profile/repoze.profile-0.6.tar.gz
++meld3 = http://www.plope.com/software/meld3/meld3-0.6.5.tar.gz
++
+ [dependencies]
+ bx_python = numpy
-style-tweaks
-trackster-styles
-phix
-bx-python-update
-functional-test-harness
-python26-new
-python26-fixes
-jqtouch-update
-tool-search-backend
 repoze.profile
-tool-search-ui
-dbkey_flexselect
+workflow-speed

tool-search-backend

 diff --git a/eggs.ini b/eggs.ini
 --- a/eggs.ini
 +++ b/eggs.ini
-@@ -50,6 +50,7 @@
+@@ -53,6 +53,7 @@
  wsgiref = 0.1.2
  Babel = 0.9.4
  wchartype = 0.1
-+Whoosh = 0.3.0b24
++Whoosh = 0.3.18
+ boto = 1.8d
  
  ; extra version information
- [tags]
-@@ -98,3 +99,4 @@
+@@ -106,4 +107,5 @@
  wsgiref = http://pypi.python.org/packages/source/w/wsgiref/wsgiref-0.1.2.zip
  Babel = http://ftp.edgewall.com/pub/babel/Babel-0.9.4.zip
  wchartype = http://ginstrom.com/code/wchartype-0.1.zip
-+Whoosh = http://pypi.python.org/packages/source/W/Whoosh/Whoosh-0.3.0b24.tar.gz
++Whoosh = http://pypi.python.org/packages/source/W/Whoosh/Whoosh-0.3.18.tar.gz
+ boto = http://boto.googlecode.com/files/boto-1.8d.tar.gz
 diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
 --- a/lib/galaxy/app.py
 +++ b/lib/galaxy/app.py
 @@ -1,6 +1,7 @@
  import sys, os, atexit
  
- from galaxy import config, jobs, util, tools, web
+ from galaxy import config, jobs, util, tools, web, cloud
 +import galaxy.tools.search
- from galaxy.tracks import store
+ ## from galaxy.tracks import store
  from galaxy.web import security
  import galaxy.model
 @@ -35,6 +36,8 @@
 --- a/lib/galaxy/web/controllers/root.py
 +++ b/lib/galaxy/web/controllers/root.py
 @@ -32,6 +32,10 @@
+         else:
              return trans.fill_template('/root/tool_menu.mako', toolbox=self.get_toolbox() )
  
-     @web.expose
++    @web.json
 +    def tool_search( self, trans, query ):
 +        return trans.app.toolbox_search.search( query )
 +
-+    @web.expose
+     @web.expose
      def tool_help( self, trans, id ):
          """Return help page for tool identified by 'id' if available"""
-         toolbox = self.get_toolbox()
-diff --git a/static/images/search_mag.png b/static/images/search_mag.png
-new file mode 100644
-index 0000000000000000000000000000000000000000..67d5ff15492d055a8bf331d957017b36d2e4b3ca
-GIT binary patch
-literal 290
-zc%17D@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`oCO|{#S9GG!XV7ZFl&wkP*AeO
-zHKHUqKdq!Zu_%?nF(p4KRlzeiF+DXXH8G{K@MNkDP|<o%7srr_TS*BE1PmStY<EwJ
-zXL!_qMCt|qk?)N^?4{nRyA|7r^`5A(vpf68;g6hWq3soE1@XD=LjCH;zSWpmR;nDn
-zzBoIj=@OSm68D8GHZ5!_92^gf9xzXu!6YWGCBWrStD--ry^WLS%i>mEzYTJpiYLzh
-z-srN@cHs;Qe+8}k&avI*s!d5M44Mb{91bfmYVMI~_@T7&WSyPe^a;r)YKx2&&Ir_c
-g6&c1DJ2WsdBnZ0Pipg5m0dzWpr>mdKI;Vst0Mqnhe*gdg
-
-diff --git a/templates/root/index.mako b/templates/root/index.mako
---- a/templates/root/index.mako
-+++ b/templates/root/index.mako
-@@ -67,7 +67,12 @@
-     <div class="unified-panel-header" unselectable="on">
-         <div class='unified-panel-header-inner'>${n_('Tools')}</div>
-     </div>
--    <div class="unified-panel-body" style="overflow: hidden;">
-+    <div class="unified-panel-subpanel" style="height: 2em; background: #999;">
-+        <div style="padding: 1px 5px;">
-+            <input type="search" placeholder="Search..." autosave="bsn_srch" results="5" style="width: 100%;"></input>
-+        </div>
-+    </div>
-+    <div class="unified-panel-body" style="overflow: hidden; top: 4em;">
-         <iframe name="galaxy_tools" src="${h.url_for( controller='root', action='tool_menu' )}" frameborder="0" style="position: absolute; margin: 0; border: 0 none; height: 100%; width: 100%;"> </iframe>
-     </div>
- </%def>
+diff --git a/templates/root/tool_menu.mako b/templates/root/tool_menu.mako
+--- a/templates/root/tool_menu.mako
++++ b/templates/root/tool_menu.mako
+@@ -85,6 +85,12 @@
+ 
+     <body class="toolMenuPage">
+         <div class="toolMenu">
++            <div id="tool-search">
++                <form action="tool_search">
++                    <input type="text" name="query">
++                </form>
++            </div>
++            
+             <div class="toolSectionList">
+                 
+                 %for key, val in toolbox.tool_panel.items():
-diff --git a/static/scripts/trackster.js b/static/scripts/trackster.js
---- a/static/scripts/trackster.js
-+++ b/static/scripts/trackster.js
-@@ -218,7 +218,6 @@
-             var y1 = data[i][1];
-             var x2 = data[i+1][0] - tile_low;
-             var y2 = data[i+1][1];
--	    console.log( x1, y1, x2, y2 );
-             // Missing data causes us to stop drawing
-             if ( isNaN( y1 ) || isNaN( y2 ) ) {
-                 in_path = false;
 diff --git a/templates/root/index.mako b/templates/root/index.mako
 --- a/templates/root/index.mako
 +++ b/templates/root/index.mako
-@@ -14,10 +14,60 @@
+@@ -66,10 +66,60 @@
      </script>
  %endif
  </%def>
+diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
+--- a/lib/galaxy/model/__init__.py
++++ b/lib/galaxy/model/__init__.py
+@@ -227,8 +227,8 @@
+     def add_dataset( self, dataset, parent_id=None, genome_build=None, set_hid = True ):
+         if isinstance( dataset, Dataset ):
+             dataset = HistoryDatasetAssociation( dataset = dataset, copied_from = dataset )
+-            object_session( self ).add( dataset )
+-            object_session( self ).flush()
++            ## object_session( self ).add( dataset )
++            ## object_session( self ).flush()
+         elif not isinstance( dataset, HistoryDatasetAssociation ):
+             raise TypeError, "You can only add Dataset and HistoryDatasetAssociation instances to a history ( you tried to add %s )." % str( dataset )
+         if parent_id:
+@@ -484,8 +484,9 @@
+         if not dataset and create_dataset:
+             # Had to pass the sqlalchemy session in order to create a new dataset
+             dataset = Dataset( state=Dataset.states.NEW )
+-            sa_session.add( dataset )
+-            sa_session.flush()
++            if sa_session:
++                sa_session.add( dataset )
++                sa_session.flush()
+         self.dataset = dataset
+         self.parent_id = parent_id
+         self.validation_errors = validation_errors
+@@ -496,8 +497,8 @@
+         return self.dataset.state
+     def set_dataset_state ( self, state ):
+         self.dataset.state = state
+-        object_session( self ).add( self.dataset )
+-        object_session( self ).flush() #flush here, because hda.flush() won't flush the Dataset object
++        ## object_session( self ).add( self.dataset )
++        ## object_session( self ).flush() #flush here, because hda.flush() won't flush the Dataset object
+     state = property( get_dataset_state, set_dataset_state )
+     def get_file_name( self ):
+         return self.dataset.get_file_name()
+diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
+--- a/lib/galaxy/tools/__init__.py
++++ b/lib/galaxy/tools/__init__.py
+@@ -1093,14 +1093,14 @@
+                 raise Exception( "Unexpected parameter type" )
+         return args
+             
+-    def execute( self, trans, incoming={}, set_output_hid=True ):
++    def execute( self, trans, incoming={}, set_output_hid=True, flush=True ):
+         """
+         Execute the tool using parameter values in `incoming`. This just
+         dispatches to the `ToolAction` instance specified by 
+         `self.tool_action`. In general this will create a `Job` that 
+         when run will build the tool's outputs, e.g. `DefaultToolAction`.
+         """
+-        return self.tool_action.execute( self, trans, incoming=incoming, set_output_hid=set_output_hid )
++        return self.tool_action.execute( self, trans, incoming=incoming, set_output_hid=set_output_hid, flush=flush )
+         
+     def params_to_strings( self, params, app ):
+         return params_to_strings( self.inputs, params, app )
+diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py
+--- a/lib/galaxy/tools/actions/__init__.py
++++ b/lib/galaxy/tools/actions/__init__.py
+@@ -12,6 +12,8 @@
+ import logging
+ log = logging.getLogger( __name__ )
+ 
++logging.getLogger('sqlalchemy.orm').setLevel(logging.DEBUG)
++
+ class ToolAction( object ):
+     """
+     The actions to be taken when a tool is run (after parameters have
+@@ -36,7 +38,9 @@
+                     formats = input.formats
+                 if data and not isinstance( data.datatype, formats ):
+                     # Need to refresh in case this conversion just took place, i.e. input above in tool performed the same conversion
+-                    trans.sa_session.refresh( data )
++                    # HACK: only do this if the dataset is already persistent, since conversion will have flushed it
++                    if data._sa_instance_state.key is not None:
++                        trans.sa_session.refresh( data )
+                     target_ext, converted_dataset = data.find_conversion_destination( formats, converter_safe = input.converter_safe( param_values, trans ) )
+                     if target_ext:
+                         if converted_dataset:
+@@ -101,7 +105,7 @@
+         tool.visit_inputs( param_values, visitor )
+         return input_datasets
+ 
+-    def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True ):
++    def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True, flush=True ):
+         def make_dict_copy( from_dict ):
+             """
+             Makes a copy of input dictionary from_dict such that all values that are dictionaries
+@@ -249,15 +253,15 @@
+                                         if check is not None:
+                                             if str( getattr( check, when_elem.get( 'attribute' ) ) ) == when_elem.get( 'value', None ):
+                                                 ext = when_elem.get( 'format', ext )
+-                    data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True, sa_session=trans.sa_session )
++                    data = trans.app.model.HistoryDatasetAssociation( extension=ext, create_dataset=True )
+                     # Commit the dataset immediately so it gets database assigned unique id
+                     trans.sa_session.add( data )
+-                    trans.sa_session.flush()
+-                    trans.app.security_agent.set_all_dataset_permissions( data.dataset, output_permissions )
++                    ### trans.sa_session.flush()
++                    ### trans.app.security_agent.set_all_dataset_permissions( data.dataset, output_permissions )
+                 # Create an empty file immediately
+-                open( data.file_name, "w" ).close()
++                ### open( data.file_name, "w" ).close()
+                 # Fix permissions
+-                util.umask_fix_perms( data.file_name, trans.app.config.umask, 0666 )
++                ### util.umask_fix_perms( data.file_name, trans.app.config.umask, 0666 )
+                 # This may not be neccesary with the new parent/child associations
+                 data.designation = name
+                 # Copy metadata from one of the inputs if requested. 
+@@ -291,21 +295,21 @@
+                 # Store output 
+                 out_data[ name ] = data
+                 # Store all changes to database
+-                trans.sa_session.flush()
++                ### trans.sa_session.flush()
+         # Add all the top-level (non-child) datasets to the history
+         for name in out_data.keys():
+             if name not in child_dataset_names and name not in incoming: #don't add children; or already existing datasets, i.e. async created
+                 data = out_data[ name ]
+                 trans.history.add_dataset( data, set_hid = set_output_hid )
+                 trans.sa_session.add( data )
+-                trans.sa_session.flush()
++                ### trans.sa_session.flush()
+         # Add all the children to their parents
+         for parent_name, child_name in parent_to_child_pairs:
+             parent_dataset = out_data[ parent_name ]
+             child_dataset = out_data[ child_name ]
+             parent_dataset.children.append( child_dataset )
+         # Store data after custom code runs 
+-        trans.sa_session.flush()
++        ### trans.sa_session.flush()
+         # Create the job object
+         job = trans.app.model.Job()
+         job.session_id = trans.get_galaxy_session().id
+@@ -332,7 +336,7 @@
+         for name, dataset in out_data.iteritems():
+             job.add_output_dataset( name, dataset )
+         trans.sa_session.add( job )
+-        trans.sa_session.flush()
++        ### trans.sa_session.flush()
+         # Some tools are not really executable, but jobs are still created for them ( for record keeping ).
+         # Examples include tools that redirect to other applications ( epigraph ).  These special tools must
+         # include something that can be retrieved from the params ( e.g., REDIRECT_URL ) to keep the job
+@@ -350,11 +354,13 @@
+             # Job should not be queued, so set state to ok
+             job.state = JOB_OK
+             job.info = "Redirected to: %s" % redirect_url
+-            trans.sa_session.add( job )
+-            trans.sa_session.flush()
++            if flush:
++                trans.sa_session.flush()
+             trans.response.send_redirect( url_for( controller='tool_runner', action='redirect', redirect_url=redirect_url ) )
+         else:
+             # Queue the job for execution
+-            trans.app.job_queue.put( job.id, tool )
+-            trans.log_event( "Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id )
++            ## trans.app.job_queue.put( job.id, tool )
++            ## trans.log_event( "Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id )
++            if flush:
++                trans.sa_session.flush()
+             return job, out_data
+diff --git a/lib/galaxy/web/controllers/workflow.py b/lib/galaxy/web/controllers/workflow.py
+--- a/lib/galaxy/web/controllers/workflow.py
++++ b/lib/galaxy/web/controllers/workflow.py
+@@ -892,7 +892,7 @@
+                                     return outputs[ conn.output_step.id ][ conn.output_name ]
+                         visit_input_values( tool.inputs, step.state.inputs, callback )
+                         # Execute it
+-                        job, out_data = tool.execute( trans, step.state.inputs )
++                        job, out_data = tool.execute( trans, step.state.inputs, flush=False )
+                         outputs[ step.id ] = out_data
+                     else:
+                         job, out_data = step.module.execute( trans, step.state )