1. Matthew Turk
  2. enzo-timing-woc

Commits

jw...@red.slac.stanford.edu  committed 0bc509c Merge

bs2

  • Participants
  • Parent commits bde9f35, abc3532
  • Branches trunk

Comments (0)

Files changed (11)

File src/enzo/Grid_Group_WriteGrid.C

View file
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
- 
+#include <math.h>
 
 
  
 void WriteListOfInts(FILE *fptr, int N, int nums[]);
 int WriteStringAttr(hid_t dset_id, char *Alabel, char *String, FILE *log_fptr);
 int FindField(int field, int farray[], int numfields);
- 
- 
+
+int GetUnits(float *DensityUnits, float *LengthUnits,
+	     float *TemperatureUnits, float *TimeUnits,
+	     float *VelocityUnits, float *MassUnits, FLOAT Time);
+
 int grid::Group_WriteGrid(FILE *fptr, char *base_name, int grid_id, HDF5_hid_t file_id)
 {
  
     /* If this is cosmology, compute the temperature field as well since
        its such a pain to compute after the fact. */
  
-    if (ComovingCoordinates || ProblemType == 62) {
+    if (OutputTemperature) {
  
       /* Allocate field and compute temperature. */
  
  
       delete temperature;
  
-    } // end: if (ComovingCoordinates)
+    } // end: if (OutputTemperature)
+
+    if (OutputCoolingTime) {
+ 
+      /* Allocate field and compute cooling time. */
+
+      float *cooling_time = new float[size];
+ 
+      float TemperatureUnits = 1, DensityUnits = 1, LengthUnits = 1,
+	VelocityUnits = 1, TimeUnits = 1, MassUnits = 1, aUnits = 1;
+      GetUnits(&DensityUnits, &LengthUnits, &TemperatureUnits,
+	       &TimeUnits, &VelocityUnits, &MassUnits, Time);
+
+      if (this->ComputeCoolingTime(cooling_time) == FAIL) {
+	fprintf(stderr, "Error in grid->ComputeCoolingTime.\n");
+	return FAIL;
+      }
+
+      // Make all cooling time values positive and convert to seconds.
+      for (i = 0;i < size;i++) {
+	cooling_time[i] = fabs(cooling_time[i]) * TimeUnits;
+      }
+ 
+      /* Copy active part of field into grid */
+ 
+      for (k = GridStartIndex[2]; k <= GridEndIndex[2]; k++)
+	for (j = GridStartIndex[1]; j <= GridEndIndex[1]; j++)
+	  for (i = GridStartIndex[0]; i <= GridEndIndex[0]; i++)
+	    temp[(i-GridStartIndex[0])                           +
+	         (j-GridStartIndex[1])*ActiveDim[0]              +
+	         (k-GridStartIndex[2])*ActiveDim[0]*ActiveDim[1] ] =
+		     io_type(
+		   cooling_time[(k*GridDimension[1] + j)*GridDimension[0] + i]
+			     );
+ 
+       file_dsp_id = H5Screate_simple((Eint32) GridRank, OutDims, NULL);
+        if (io_log) fprintf(log_fptr, "H5Screate file_dsp_id: %"ISYM"\n", file_dsp_id);
+        if( file_dsp_id == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      if (io_log) fprintf(log_fptr,"H5Dcreate with Name = Cooling_Time\n");
+ 
+      dset_id = H5Dcreate(group_id, "Cooling_Time", file_type_id, file_dsp_id, H5P_DEFAULT);
+        if (io_log) fprintf(log_fptr, "H5Dcreate id: %"ISYM"\n", dset_id);
+        if( dset_id == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      if ( DataUnits[field] == NULL )
+      {
+        DataUnits[field] = "none";
+      }
+ 
+      WriteStringAttr(dset_id, "Label", "Temperature", log_fptr);
+      WriteStringAttr(dset_id, "Units", "K", log_fptr);
+      WriteStringAttr(dset_id, "Format", "e10.4", log_fptr);
+      WriteStringAttr(dset_id, "Geometry", "Cartesian", log_fptr);
+ 
+      h5_status = H5Dwrite(dset_id, float_type_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (VOIDP) temp);
+        if (io_log) fprintf(log_fptr, "H5Dwrite: %"ISYM"\n", h5_status);
+        if( h5_status == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      h5_status = H5Sclose(file_dsp_id);
+        if (io_log) fprintf(log_fptr, "H5Sclose: %"ISYM"\n", h5_status);
+        if( h5_status == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      h5_status = H5Dclose(dset_id);
+        if (io_log) fprintf(log_fptr, "H5Dclose: %"ISYM"\n", h5_status);
+        if( h5_status == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      delete cooling_time;
+ 
+    } // if (OutputCoolingTime)
  
     /* Make sure that there is a copy of dark matter field to save
        (and at the right resolution). */

File src/enzo/Grid_WriteGrid.C

View file
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
- 
+#include <math.h>
 
 
  
 int WriteStringAttr(hid_t dset_id, char *Alabel, char *String, FILE *log_fptr);
  
 int FindField(int field, int farray[], int numfields);
- 
+
+int GetUnits(float *DensityUnits, float *LengthUnits,
+	     float *TemperatureUnits, float *TimeUnits,
+	     float *VelocityUnits, float *MassUnits, FLOAT Time);
  
 int grid::WriteGrid(FILE *fptr, char *base_name, int grid_id)
 {
     /* If this is cosmology, compute the temperature field as well since
        its such a pain to compute after the fact. */
  
-    if (ComovingCoordinates || ProblemType == 62) {
+    if (OutputTemperature) {
  
       /* Allocate field and compute temperature. */
  
  
       delete temperature;
  
-    } // end: if (ComovingCoordinates)
+    } // end: if (OutputTemperature)
+
+    if (OutputCoolingTime) {
+ 
+      /* Allocate field and compute cooling time. */
+
+      float *cooling_time = new float[size];
+ 
+      float TemperatureUnits = 1, DensityUnits = 1, LengthUnits = 1,
+	VelocityUnits = 1, TimeUnits = 1, MassUnits = 1, aUnits = 1;
+      GetUnits(&DensityUnits, &LengthUnits, &TemperatureUnits,
+	       &TimeUnits, &VelocityUnits, &MassUnits, Time);
+
+      if (this->ComputeCoolingTime(cooling_time) == FAIL) {
+	fprintf(stderr, "Error in grid->ComputeCoolingTime.\n");
+	return FAIL;
+      }
+
+      // Make all cooling time values positive and convert to seconds.
+      for (i = 0;i < size;i++) {
+	cooling_time[i] = fabs(cooling_time[i]) * TimeUnits;
+      }
+ 
+      /* Copy active part of field into grid */
+ 
+      for (k = GridStartIndex[2]; k <= GridEndIndex[2]; k++)
+	for (j = GridStartIndex[1]; j <= GridEndIndex[1]; j++)
+	  for (i = GridStartIndex[0]; i <= GridEndIndex[0]; i++)
+	    temp[(i-GridStartIndex[0])                           +
+	         (j-GridStartIndex[1])*ActiveDim[0]              +
+	         (k-GridStartIndex[2])*ActiveDim[0]*ActiveDim[1] ] =
+		     io_type(
+		   cooling_time[(k*GridDimension[1] + j)*GridDimension[0] + i]
+			     );
+ 
+ 
+      file_dsp_id = H5Screate_simple((Eint32) GridRank, OutDims, NULL);
+        if (io_log) fprintf(log_fptr, "H5Screate file_dsp_id: %"ISYM"\n", file_dsp_id);
+        if( file_dsp_id == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      if (io_log) fprintf(log_fptr,"H5Dcreate with Name = Cooling_Time\n");
+ 
+      dset_id = H5Dcreate(file_id, "Cooling_Time", file_type_id, file_dsp_id, H5P_DEFAULT);
+        if (io_log) fprintf(log_fptr, "H5Dcreate id: %"ISYM"\n", dset_id);
+        if( dset_id == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      if ( DataUnits[field] == NULL )
+      {
+        DataUnits[field] = "none";
+      }
+ 
+      WriteStringAttr(dset_id, "Label", "Cooling_Time", log_fptr);
+      WriteStringAttr(dset_id, "Units", "s", log_fptr);
+      WriteStringAttr(dset_id, "Format", "e10.4", log_fptr);
+      WriteStringAttr(dset_id, "Geometry", "Cartesian", log_fptr);
+ 
+      h5_status = H5Dwrite(dset_id, float_type_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (VOIDP) temp);
+        if (io_log) fprintf(log_fptr, "H5Dwrite: %"ISYM"\n", h5_status);
+        if( h5_status == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      h5_status = H5Sclose(file_dsp_id);
+        if (io_log) fprintf(log_fptr, "H5Sclose: %"ISYM"\n", h5_status);
+        if( h5_status == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      h5_status = H5Dclose(dset_id);
+        if (io_log) fprintf(log_fptr, "H5Dclose: %"ISYM"\n", h5_status);
+        if( h5_status == h5_error ){my_exit(EXIT_FAILURE);}
+ 
+      delete cooling_time;
+ 
+    } // if (OutputCoolingTime)
  
     /* Make sure that there is a copy of dark matter field to save
        (and at the right resolution). */

File src/enzo/Grid_WriteGridX.C

View file
     /* If this is cosmology, compute the temperature field as well since
        its such a pain to compute after the fact. */
  
-    if (ComovingCoordinates) {
+    if (OutputTemperature) {
  
       /* Allocate field and compute temperature. */
  

File src/enzo/Make.config.assemble

View file
 	$(error Illegal value '$(CONFIG_TPVEL)' for $$(CONFIG_TPVEL))
     endif
 
+#-----------------------------------------------------------------------
+# DETERMINE LOAD BALANCING SETTINGS
+#-----------------------------------------------------------------------
+
+    ERROR_LOAD_BALANCE = 1
+
+    # Settings to turn isolated gravity boundary conditions ON
+
+    ifeq ($(CONFIG_LOAD_BALANCE),yes)
+        ERROR_LOAD_BALANCE = 0
+        ASSEMBLE_LOAD_BALANCE_DEFINES = -DENABLE_LOAD_BALANCE
+    endif
+
+    # Settings to turn isolated gravity boundary conditions OFF
+
+    ifeq ($(CONFIG_LOAD_BALANCE),no)
+        ERROR_LOAD_BALANCE = 0
+        ASSEMBLE_LOAD_BALANCE_DEFINES =
+    endif
+
+    # error if CONFIG_LOAD_BALANCE is incorrect
+
+    ifeq ($(ERROR_LOAD_BALANCE),1)
+       .PHONY: error_load_balance
+       error_load_balance:
+	$(error Illegal value '$(CONFIG_LOAD_BALANCE)' for $$(CONFIG_LOAD_BALANCE))
+    endif
+
 
 #-----------------------------------------------------------------------
 # DETERMINE RADIATIVE TRANSFER SETTINGS
               $(ASSEMBLE_ISOBCS_DEFINES) \
               $(ASSEMBLE_TPVEL_DEFINES) \
               $(ASSEMBLE_PHOTON_DEFINES) \
+              $(ASSEMBLE_LOAD_BALANCE_DEFINES) \
               $(ASSEMBLE_UNIGRID_TRANSPOSE_DEFINES)
 
     INCLUDES = $(MACH_INCLUDES) \
-               $(ASSEMBLE_MPI_INCLUDES) \
+    	       $(ASSEMBLE_MPI_INCLUDES) \
                $(ASSEMBLE_HYPRE_INCLUDES) \
                $(ASSEMBLE_JBPERF_INCLUDES) \
                $(ASSEMBLE_PAPI_INCLUDES) \

File src/enzo/Make.config.settings

View file
 
      CONFIG_PHOTON = no
 
+#======================================================================= 
+# CONFIG_LOAD_BALANCE 
+#======================================================================= 
+#    yes           Use round-robin load balancing of grids 
+#    no            Do not use load balancing 
+#----------------------------------------------------------------------- 
+ 
+     CONFIG_LOAD_BALANCE = yes
+

File src/enzo/Make.config.targets

View file
 	@echo "      gmake photon-yes"
 	@echo "      gmake photon-no"
 	@echo 
+	@echo "   Set whether or not to use load balancing"
+	@echo
+	@echo "      gmake load-balance-yes"
+	@echo "      gmake load-balance-no"
+	@echo 
 
 #-----------------------------------------------------------------------
 
 	@echo "   CONFIG_ISOBCS:                $(CONFIG_ISOBCS)"
 	@echo "   CONFIG_TPVEL:                 $(CONFIG_TPVEL)"
 	@echo "   CONFIG_PHOTON:                $(CONFIG_PHOTON)"
+	@echo "   CONFIG_LOAD_BALANCE:          $(CONFIG_LOAD_BALANCE)" 
 	@echo
 
 #-----------------------------------------------------------------------
 	$(MAKE) show-config | grep CONFIG_PHOTON:
 	@echo
 
+#-----------------------------------------------------------------------
+
+VALID_LOAD_BALANCE = load-balance-yes load-balance-no
+.PHONY: $(VALID_LOAD_BALANCE)
+
+load-balance-yes: CONFIG_LOAD_BALANCE-yes
+load-balance-no: CONFIG_LOAD_BALANCE-no
+load-balance-%:
+	@printf "\n\tInvalid target: $@\n\n\tValid targets: [$(VALID_LOAD_BALANCE)]\n\n"
+CONFIG_LOAD_BALANCE-%:
+	@tmp=.config.temp; \
+        grep -v CONFIG_LOAD_BALANCE $(MAKE_CONFIG_OVERRIDE) > $${tmp}; \
+        mv $${tmp} $(MAKE_CONFIG_OVERRIDE); \
+        echo "CONFIG_LOAD_BALANCE = $*" >> $(MAKE_CONFIG_OVERRIDE); \
+	$(MAKE) show-config | grep CONFIG_LOAD_BALANCE:
+	@echo

File src/enzo/ReadParameterFile.C

View file
         return FAIL;
       }
     }
+
+    ret += sscanf(line, "LoadBalancing = %"ISYM, &LoadBalancing);
  
     if (sscanf(line, "TimeActionType[%"ISYM"] = %"ISYM, &dim, &int_dummy) == 2) {
       ret++; TimeActionType[dim] = int_dummy;
     if (sscanf(line, "CoolDataParameterFile = %s", dummy) == 1)
       CoolData.ParameterFilename = dummy;
 
+    ret += sscanf(line, "OutputCoolingTime = %"ISYM, &OutputCoolingTime);
+    ret += sscanf(line, "OutputTemperature = %"ISYM, &OutputTemperature);
+
     ret += sscanf(line, "ZEUSQuadraticArtificialViscosity = %"FSYM,
 		  &ZEUSQuadraticArtificialViscosity);
     ret += sscanf(line, "ZEUSLinearArtificialViscosity = %"FSYM,

File src/enzo/RebuildHierarchy.C

View file
       FastSiblingLocatorFinalize(&ChainingMesh);
  
       /* Redistribute grids over processors to Load balance. */
-#ifdef ENABLE_LOAD_BALANCE 
-      TIME_MSG("Load balancing");
-      CommunicationLoadBalanceGrids(SubgridHierarchyPointer, subgrids, 
-				    MoveParticles);
-      TIME_MSG("Finished load balancing");
-#endif
+      switch( LoadBalancing ){
+      case 1:
+	TIME_MSG("Load balancing");
+	CommunicationLoadBalanceGrids(SubgridHierarchyPointer, subgrids, 
+				      MoveParticles);
+	TIME_MSG("Finished load balancing");
+	break;
+      default:
+	
+	break;
+      }
+
 
       /* If this is the finest level with static subgrids, the grids
 	 should be distributed enough to collect the particles on each

File src/enzo/SetDefaultGlobalValues.C

View file
  
   MetaData.LocalDir            = NULL;
   MetaData.GlobalDir           = NULL;
+
+  LoadBalancing = 1; //On, memory equalization method
  
   for (i = 0; i < MAX_TIME_ACTIONS; i++) {
     TimeActionType[i]      = 0;
   CoolData.alpha0             = 1.5;               // radiation spectral slope
   CoolData.f3                 = 1.0e-21;           // radiation normalization
   CoolData.ParameterFilename  = NULL;
- 
+
+  OutputCoolingTime = 0;
+  OutputTemperature = 0;
+
   ZEUSLinearArtificialViscosity    = 0.0;
   ZEUSQuadraticArtificialViscosity = 2.0;
   UseMinimumPressureSupport        = FALSE;

File src/enzo/WriteParameterFile.C

View file
   for (dim = 0; dim < MAX_CUBE_DUMPS; dim++)
     if (CubeDumps[dim] != NULL)
       fprintf(fptr, "CubeDump[%"ISYM"]            = %s\n", dim, CubeDumps[dim]);
+
+  fprintf(fptr,"LoadBalancing         = %"ISYM"\n",LoadBalancing);
  
   for (dim = 0; dim < MAX_TIME_ACTIONS; dim++)
     if (TimeActionType[dim] > 0) {
 
   if (CoolData.ParameterFilename != NULL)
     fprintf(fptr, "CoolDataParameterFile = %s\n\n", CoolData.ParameterFilename);
+
+  fprintf(fptr, "OutputCoolingTime              = %"ISYM"\n", OutputCoolingTime);
+  fprintf(fptr, "OutputTemperature              = %"ISYM"\n", OutputTemperature);
  
   fprintf(fptr, "ZEUSLinearArtificialViscosity    = %"GSYM"\n",
 	  ZEUSLinearArtificialViscosity);

File src/enzo/global_data.h

View file
 # define EXTERN extern
 #endif
 
+/* Load Balancing.  Currently only memory count method implemented
+                          0 = off
+                          1 = Equalize processor memory count */
+EXTERN int LoadBalancing;
+
 /* debugging, extraction flags */
 
 EXTERN int debug;
 
 EXTERN int CellFlaggingMethod[MAX_FLAGGING_METHODS];
 
+
 /* Velocity to limit timesteps */
 
 EXTERN float TimestepSafetyVelocity;
 EXTERN int RadiationFieldLevelRecompute;
 EXTERN int RadiationXRaySecondaryIon;
 
+/* Output cooling time with grid data. */
+
+EXTERN int OutputCoolingTime;
+
+/* Output temperature with grid data. */
+
+EXTERN int OutputTemperature;
+
 /* ZEUS Hydro artificial viscosity parameters (C1, C2 of Stone & Norman). */
 
 EXTERN float ZEUSLinearArtificialViscosity;