Matthew Turk committed 7044e4e Merge

Merged in brittonsmith/yt (pull request #393: Making simulation time series work better in parallel.)

Comments (0)

Files changed (1)


     InvalidSimulationTimeSeries, \
     MissingParameter, \
+from yt.utilities.parallel_tools.parallel_analysis_interface import \
+    parallel_objects
 from yt.convenience import \
         if not my_all_outputs:
             TimeSeriesData.__init__(self, outputs=[], parallel=parallel)
-  "%d outputs loaded into time series." % 0)
+  "0 outputs loaded into time series.")
         # Apply selection criteria to the set.
         TimeSeriesData.__init__(self, outputs=init_outputs, parallel=parallel)
-"%d outputs loaded into time series." % len(init_outputs))
+"%d outputs loaded into time series.", len(init_outputs))
     def _parse_parameter_file(self):
         elif self.parameters['dtDataDump'] > 0 and \
           self.parameters['CycleSkipDataDump'] > 0:
-  "Simulation %s has both dtDataDump and CycleSkipDataDump set." % self.parameter_filename )
+  "Simulation %s has both dtDataDump and CycleSkipDataDump set.", self.parameter_filename )
   "    Unable to calculate datasets.  Attempting to search in the current directory")
                     'StopCycle' in self.parameters):
                 raise NoStoppingCondition(self.parameter_filename)
             if self.final_time is None:
-                mylog.warn('Simulation %s has no stop time set, stopping condition will be based only on cycles.' %
+                mylog.warn('Simulation %s has no stop time set, stopping condition will be based only on cycles.',
     def _set_parameter_defaults(self):
         self.all_outputs = self.all_time_outputs + self.all_redshift_outputs
         self.all_outputs.sort(key=lambda obj: obj['time'])
-"Located %d total outputs." % len(self.all_outputs))
+        only_on_root(, "Located %d total outputs.", len(self.all_outputs))
         # manually set final time and redshift with last output
         if self.all_outputs:
     def _check_for_outputs(self, potential_outputs):
         r"""Check a list of files to see if they are valid datasets."""
-"Checking %d potential outputs." %
-                   len(potential_outputs))
+        only_on_root(, "Checking %d potential outputs.", 
+                     len(potential_outputs))
-        my_outputs = []
-        for output in potential_outputs:
+        my_outputs = {}
+        for my_storage, output in parallel_objects(potential_outputs, 
+                                                   storage=my_outputs):
             if self.parameters['DataDumpDir'] in output:
                 dir_key = self.parameters['DataDumpDir']
                 output_key = self.parameters['DataDumpName']
                     pf = load(filename)
                     if pf is not None:
-                        my_outputs.append({'filename': filename,
-                                           'time': pf.current_time})
+                        my_storage.result = {'filename': filename,
+                                             'time': pf.current_time}
                         if pf.cosmological_simulation:
-                            my_outputs[-1]['redshift'] = pf.current_redshift
+                            my_storage.result['redshift'] = pf.current_redshift
                 except YTOutputNotIdentified:
-                    mylog.error('Failed to load %s' % filename)
+                    mylog.error('Failed to load %s', filename)
+        my_outputs = [my_output for my_output in my_outputs.values() \
+                      if my_output is not None]
         return my_outputs
                     and outputs[0] not in my_outputs:
-                mylog.error("No dataset added for %s = %f." % (key, value))
+                mylog.error("No dataset added for %s = %f.", key, value)
         outputs.sort(key=lambda obj: obj['time'])
         return my_outputs
         r"""Write cosmology output parameters for a cosmology splice.
-"Writing redshift output list to %s." % filename)
+"Writing redshift output list to %s.", filename)
         f = open(filename, 'w')
         for q, output in enumerate(outputs):
             z_string = "%%s[%%d] = %%.%df" % decimals