Commits

Anonymous committed 67dc353 Merge

merge

  • Participants
  • Parent commits d659969, b9129ea

Comments (0)

Files changed (31)

File regression_tests/Makefile

 	-find . -type f -name '*.stdout' -print0 | xargs -0 rm
 	-find . -type f -name '*.old' -print0 | xargs -0 rm
 	-find . -type f -name '*~' -print0 | xargs -0 rm
+	-find . -type f -name 'tmp-restart-*' -print0 | xargs -0 rm
+	-find . -type f -name '*.chk' -print0 | xargs -0 rm
 	-rm -f default/anisothermal/thc_1d.h5
 	-rm -f default/anisothermal/th_1d.h5
 	-rm -f mfd/mfd-mas.dat
 	-rm -f mfd/mfd.h5
 	-rm -f scco2/scco2.h5
 	-rm -f scco2/scco2-mas.dat
-	-rm -f default/543/543_flow_and_tracer_restart-np8-restart.chk
-	-rm -f default/543/543_flow_and_tracer_restart-restart.chk

File regression_tests/default/543/543.cfg

 [suites]
 flow = 543_flow
-transport = 543_flow_and_tracer 543_flow_and_tracer_restart 543_flow_and_tracer_init_to_ss 543_flow_and_tracer_run_as_ss
+transport = 543_flow_and_tracer 543_flow_and_tracer_init_to_ss 543_flow_and_tracer_run_as_ss
 geochemistry = 543_hanford_srfcplx_base 543_hanford_srfcplx_param 543_hanford_srfcplx_mr
-parallel = 543_flow-np8 543_flow_and_tracer-np8 543_hanford_srfcplx_param-np8 543_flow_and_tracer_restart-np8
+parallel = 543_flow-np8 543_flow_and_tracer-np8 543_hanford_srfcplx_param-np8
 
 standard = 543_flow 
            543_flow_and_tracer
-           543_flow_and_tracer_restart
            543_flow_and_tracer_init_to_ss
            543_flow_and_tracer_run_as_ss
            543_hanford_srfcplx_base
            543_hanford_srfcplx_mr
 standard_parallel = 543_flow-np8
                     543_flow_and_tracer-np8
-                    543_flow_and_tracer_restart-np8
                     543_hanford_srfcplx_param-np8
 
 [default-test-criteria]
 np=8
 
 [543_flow_and_tracer]
+restart_timestep = 10
 
 [543_flow_and_tracer-np8]
+#restart_timestep = 10
 np=8
 
 [543_hanford_srfcplx_base]
+#restart_timestep = 10
 
 [543_hanford_srfcplx_mr]
 
 [543_hanford_srfcplx_param-np8]
 np=8
 
-[543_flow_and_tracer_restart]
-
-[543_flow_and_tracer_restart-np8]
-np=8
-
 [543_flow_and_tracer_init_to_ss]
 
 [543_flow_and_tracer_run_as_ss]

File regression_tests/default/543/543_flow_and_tracer-np8.in

   MATERIAL ./543.h5
 END
 
+:=========================== checkpoint / restart =============================
+CHECKPOINT 10
+

File regression_tests/default/543/543_flow_and_tracer.in

   MATERIAL ./543.h5
 END
 
+:=========================== checkpoint / restart =============================
+CHECKPOINT 10
+

File regression_tests/default/543/543_flow_and_tracer_restart-np8.chk30

Binary file removed.

File regression_tests/default/543/543_flow_and_tracer_restart-np8.in

-:Description: 3D test problem for testing chemical reactions
-:             based on chemistry from Hammond and Lichtner, WRR, 2010
-
-:=========================== regression =======================================
-REGRESSION
-  CELLS_PER_PROCESS 2
-  CELLS
-    29
-  /
-END
-
-:=========================== flow mode ========================================
-MODE RICHARDS
-CHECKPOINT 30
-RESTART 543_flow_and_tracer_restart-np8.chk30
-
-:=========================== chemistry ========================================
-CHEMISTRY
-  PRIMARY_SPECIES
-  Tracer
-  Tracer2
-  /
-  OUTPUT
-    All
-  /
-END
-
-:=========================== solver options ===================================
-TIMESTEPPER
-  TS_ACCELERATION 8
-END
-
-NEWTON_SOLVER FLOW
-END
-
-LINEAR_SOLVER FLOW
-END
-
-NEWTON_SOLVER TRANSPORT
-END
-
-LINEAR_SOLVER TRANSPORT
-END
-
-:=========================== discretization ===================================
-GRID
-  TYPE structured
-  ORIGIN 0.d0 0.d0 0.d0
-  NXYZ 5 4 3
-  DXYZ
-    10. 11. 12. 13. 14.
-    13. 12. 11. 10.
-    15. 20. 25.
-  /
-END
-
-:=========================== fluid properties =================================
-FLUID_PROPERTY 
-  DIFFUSION_COEFFICIENT 1.d-9
-END
-
-:=========================== datasets =========================================
-skip
-DATASET U
-  HDF5_DATASET_NAME U 
-  FILENAME datasets.h5
-END
-
-DATASET Calcite
-  HDF5_DATASET_NAME Calcite 
-  FILENAME datasets.h5
-END
-noskip
-
-:=========================== material properties ==============================
-MATERIAL_PROPERTY soil1
-  ID 1
-  POROSITY 0.25d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf1
-  PERMEABILITY
-    PERM_X 1.d-12
-    PERM_Y 1.d-12
-    PERM_Z 1.d-13
-  /
-END
-
-MATERIAL_PROPERTY soil2
-  ID 2
-  POROSITY 0.25d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf2
-  PERMEABILITY
-    PERM_X 1.d-11
-    PERM_Y 1.d-11
-    PERM_Z 1.d-12
-  /
-END
-
-MATERIAL_PROPERTY soil3
-  ID 3
-  POROSITY 0.2d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf3
-  PERMEABILITY
-    PERM_X 1.d-10
-    PERM_Y 1.d-10
-    PERM_Z 1.d-11
-  /
-END
-
-MATERIAL_PROPERTY soil4
-  ID 4
-  POROSITY 0.25d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf4
-  PERMEABILITY
-    PERM_X 1.d-9
-    PERM_Y 1.d-9
-    PERM_Z 1.d-10
-  /
-END
-
-:=========================== saturation functions =============================
-SATURATION_FUNCTION sf1
-  SATURATION_FUNCTION_TYPE VAN_GENUCHTEN
-  RESIDUAL_SATURATION 0.115
-  LAMBDA 0.286
-  ALPHA 1.9401d-4
-END
-
-SATURATION_FUNCTION sf2
-  SATURATION_FUNCTION_TYPE VAN_GENUCHTEN
-  RESIDUAL_SATURATION 0.110
-  LAMBDA 0.486
-  ALPHA 3.8801d-4
-END
-
-SATURATION_FUNCTION sf3
-  SATURATION_FUNCTION_TYPE VAN_GENUCHTEN
-  RESIDUAL_SATURATION 0.118
-  LAMBDA 0.541
-  ALPHA 1.0211d-4
-END
-
-SATURATION_FUNCTION sf4
-  SATURATION_FUNCTION_TYPE VAN_GENUCHTEN
-  RESIDUAL_SATURATION 0.143
-  LAMBDA 0.527
-  ALPHA 5.1054d-5
-END
-
-:=========================== output options ===================================
-OUTPUT
-:  TIMES d 0.001 0.01 0.1 1. 10. 
-:  PERIODIC TIME 1 d
-  FORMAT TECPLOT BLOCK
-  VELOCITIES
-END
-
-:=========================== times ============================================
-TIME
-  FINAL_TIME 100.d0 d
-  INITIAL_TIMESTEP_SIZE 1.d-6 d
-  MAXIMUM_TIMESTEP_SIZE 10.d0 d
-END
-
-:=========================== regions ==========================================
-REGION all
-  COORDINATES
-    0.d0 0.d0 0.d0
-    60.d0 46.d0 60.d0
-  /
-END
-
-REGION top
-  FACE TOP
-  COORDINATES
-    0.d0 0.d0 60.d0
-    60.d0 46.d0 60.d0
-  /
-END
-
-REGION bottom
-  FACE BOTTOM
-  COORDINATES
-    0.d0 0.d0 0.d0
-    60.d0 46.d0 0.d0
-  /
-END
-
-REGION west
-  FACE WEST
-  COORDINATES
-    0.d0 0.d0 0.d0
-    0.d0 46.d0 60.d0
-  /
-END
-
-REGION east
-  FACE EAST
-  COORDINATES
-    60.d0 0.d0 0.d0
-    60.d0 46.d0 60.d0
-  /
-END
-
-REGION north
-  FACE NORTH
-  COORDINATES
-    0.d0 46.d0 0.d0
-    60.d0 46.d0 60.d0
-  /
-END
-
-REGION south
-  FACE SOUTH
-  COORDINATES
-    0.d0 0.d0 0.d0
-    60.d0 0.d0 60.d0
-  /
-END
-
-REGION plume
-  BLOCK 2 3 2 3 2 3
-END
-
-REGION well
-  BLOCK 4 4 2 3 3 3 
-END
-
-:=========================== flow conditions ==================================
-FLOW_CONDITION initial
-  TYPE
-    PRESSURE hydrostatic
-  /
-  DATUM 0.d0 0.d0 35.d0
-  GRADIENT
-    PRESSURE -1.6666667d-2 0.d0 0.d0
-  /
-  PRESSURE 101325 ! Pa
-END
-
-FLOW_CONDITION east
-  TYPE
-    PRESSURE hydrostatic
-  /
-  INTERPOLATION LINEAR
-  DATUM LIST
-    TIME_UNITS d
-    0.d0 0.d0 0.d0 34.d0
-    10.d0 0.d0 0.d0 39.d0
-    50.d0 0.d0 0.d0 33.d0
-    100.d0 0.d0 0.d0 34.d0
-  /
-  PRESSURE 101325 ! Pa
-END
-
-FLOW_CONDITION recharge
-  TYPE
-    FLUX neumann
-  /
-  FLUX 5.d0 cm/yr
-END
-
-FLOW_CONDITION injection_well
-  TYPE
-    RATE scaled_volumetric_rate
-  /
-  RATE 1 m^3/hr
-END
-
-:=========================== transport conditions =============================
-TRANSPORT_CONDITION columbia_river
-  TYPE dirichlet_zero_gradient
-  CONSTRAINT_LIST
-    0.d0 river_water
-  /
-END
-
-TRANSPORT_CONDITION groundwater
-  TYPE dirichlet_zero_gradient
-  CONSTRAINT_LIST
-    0.d0 groundwater
-  /
-END
-
-TRANSPORT_CONDITION tracer_pulse
-  TYPE dirichlet
-  TIME_UNITS d
-  CONSTRAINT_LIST
-    0.d0 well_tracer
-    10.d0 groundwater
-  /
-END
-
-TRANSPORT_CONDITION rainwater
-  TYPE dirichlet
-  CONSTRAINT_LIST
-    0.d0 groundwater
-  /
-END
-
-TRANSPORT_CONDITION source_zone
-  TYPE dirichlet
-  CONSTRAINT_LIST
-    0.d0 U_source
-  /
-END
-
-:=========================== transport constraints ============================
-CONSTRAINT groundwater
-  CONCENTRATIONS
-    Tracer   1.e-7            F
-    Tracer2  1.e-7            F
-  /
-END
-
-CONSTRAINT U_source
-  CONCENTRATIONS
-    Tracer   1.e-7            F
-    Tracer2  1.e-7            F
-  /
-END
-
-CONSTRAINT river_water
-  CONCENTRATIONS
-    Tracer   1.e-3            F
-    Tracer2  1.e-7            F
-  /
-END
-
-CONSTRAINT well_tracer
-  CONCENTRATIONS
-    Tracer   1.e-7            F
-    Tracer2  1.e-3            F
-  /
-END
-
-:=========================== condition couplers ===============================
-: initial condition
-INITIAL_CONDITION
-  FLOW_CONDITION initial
-  TRANSPORT_CONDITION groundwater
-  REGION all
-END
-
-INITIAL_CONDITION
-  FLOW_CONDITION initial
-  TRANSPORT_CONDITION source_zone
-  REGION plume
-END
-
-: west boundary condition
-BOUNDARY_CONDITION west
-  FLOW_CONDITION initial
-  TRANSPORT_CONDITION groundwater
-  REGION west
-END
-
-: east boundary condition
-BOUNDARY_CONDITION east
-  FLOW_CONDITION east
-  TRANSPORT_CONDITION columbia_river
-  REGION east
-END
-
-: top boundary condition
-BOUNDARY_CONDITION top
-  FLOW_CONDITION recharge
-  TRANSPORT_CONDITION rainwater
-  REGION top
-END
-
-: well source/sink
-SOURCE_SINK well
-  FLOW_CONDITION injection_well
-  TRANSPORT_CONDITION tracer_pulse
-  REGION well
-END
-
-:=========================== stratigraphy couplers ============================
-STRATA
-  MATERIAL ./543.h5
-END
-

File regression_tests/default/543/543_flow_and_tracer_restart-np8.regression.gold

--- PRESSURE: Liquid Pressure --
-      Max:   3.6989366949270E+05
-      Min:  -1.1821105596033E+04
-     Mean:   1.9217675569995E+05
-       29:   1.8951454147316E+05
-        1:   3.6935555583801E+05
-       21:   1.9890793169260E+05
-        4:   3.6254807884526E+05
-       24:   1.8942725148647E+05
-       11:   3.6989366949270E+05
-       31:   1.9891824328610E+05
-       14:   3.6194854335210E+05
-       34:   1.8975101022565E+05
-       41:  -9.4594317512418E+03
-       46:  -1.1821105596033E+04
-       44:   1.0771949632254E+04
-       49:   2.0587526367949E+04
-       51:  -4.6007929700821E+03
-       56:  -1.1164255534721E+04
-       54:   6.1962723112398E+04
-       59:   5.2857080558070E+04
--- SATURATION: Liquid Saturation --
-      Max:   1.0000000000000E+00
-      Min:   1.3540404768644E-01
-     Mean:   7.3892947317230E-01
-       29:   1.0000000000000E+00
-        1:   1.0000000000000E+00
-       21:   1.0000000000000E+00
-        4:   1.0000000000000E+00
-       24:   1.0000000000000E+00
-       11:   1.0000000000000E+00
-       31:   1.0000000000000E+00
-       14:   1.0000000000000E+00
-       34:   1.0000000000000E+00
-       41:   1.3540404768644E-01
-       46:   1.6717842418357E-01
-       44:   2.9551652111505E-01
-       49:   3.1537359949292E-01
-       51:   1.7113172315466E-01
-       56:   1.6751541927347E-01
-       54:   2.8484418849342E-01
-       59:   2.4975575990074E-01
--- CONCENTRATION: Total Tracer --
-      Max:   4.2831032235066E-04
-      Min:   1.0110785748917E-07
-     Mean:   6.8506827287310E-05
-       29:   1.1912360436942E-04
-        1:   1.0675826656488E-06
-       21:   1.9889764660744E-05
-        4:   4.1660668847528E-05
-       24:   3.2971303317418E-04
-       11:   1.6910506310990E-05
-       31:   5.3944919665200E-07
-       14:   2.5755851469870E-04
-       34:   3.4307209392662E-05
-       41:   1.0110785748917E-07
-       46:   1.0530535632110E-07
-       44:   3.3473797711365E-07
-       49:   1.0153427523527E-07
-       51:   1.4960532114759E-07
-       56:   2.6148622889419E-07
-       54:   1.0134705407573E-07
-       59:   5.7836951173590E-06
--- CONCENTRATION: Total Tracer2 --
-      Max:   6.9603772176417E-05
-      Min:   1.0001745988507E-07
-     Mean:   8.2896575215877E-06
-       29:   1.8697778492552E-07
-        1:   1.0003327257109E-07
-       21:   1.0135762408820E-07
-        4:   1.0043332857529E-07
-       24:   1.7588925705946E-07
-       11:   1.0017363673827E-07
-       31:   1.0005267145599E-07
-       14:   1.0355613080754E-07
-       34:   1.2930603845876E-07
-       41:   1.0023990501074E-07
-       46:   1.3169214281569E-07
-       44:   5.5481935951355E-05
-       49:   1.3190261804958E-05
-       51:   5.4486854340903E-07
-       56:   1.1319641347265E-07
-       54:   1.5124094371736E-05
-       59:   6.9603772176417E-05
--- DISCRETE: Material ID --
-      Max:         4
-      Min:         1
-     Mean:   2.3333333333333E+00
-       29:         3
-        1:         1
-       21:         2
-        4:         1
-       24:         2
-       11:         2
-       31:         3
-       14:         2
-       34:         3
-       41:         2
-       46:         3
-       44:         4
-       49:         4
-       51:         3
-       56:         3
-       54:         3
-       59:         3
--- GENERIC: LIQUID VELOCITY [m/d] --
-       29:   1.6066524082348E-01 -1.0572585837951E-01  2.6592667180452E-04
-        1:   1.8283368317901E-02 -1.0872639333325E-03 -7.9798244115749E-04
-       21:   3.5716648053043E-02  1.1938351754432E-02 -3.9899166299002E-04
-        4:   1.5647726193079E-02  3.6531452323675E-03  2.2543503604957E-03
-       24:   5.0659558054405E-02 -1.1929478516395E-02  1.0933782655567E-03
-       11:   5.2986604383190E-02  1.5253602752422E-02 -1.9549487765512E-03
-       31:   3.1866628834196E-01  4.8193171396413E-02 -9.7826575131334E-04
-       14:   6.3010075495963E-02 -2.3907109561846E-02  1.2003649154151E-02
-       34:   4.1081852569142E-01 -1.5548365607256E-01  5.4530661990800E-03
-       41:  -2.5078127396855E-06  5.6772429322464E-09 -6.8493593096207E-05
-       46:  -5.7128773797269E-05 -6.7672966439869E-06 -6.8558650871244E-05
-       44:   1.0860316982516E-02 -2.2077823543707E-02 -1.0229006537607E-04
-       49:   2.2563318243679E-02 -2.0117965320514E-02 -7.0708722658542E-04
-       51:  -2.0660378037848E-04 -2.9778293132716E-08 -6.9284513722691E-05
-       56:  -8.8631185906198E-05  1.3480713944641E-05 -6.8852399892491E-05
-       54:   9.4826971284253E-04 -7.9182689438773E-03 -6.1725152868035E-04
-       59:   3.0924645431568E-04  2.3215692095667E-03 -2.5042975913067E-04
--- SOLUTION: Flow --
-   Time (seconds):   5.7351589202881E-03
-   Time Steps:           34
-   Newton Iterations:           92
-   Solver Iterations:          467
-   Time Step Cuts:            0
-   Solution 2-Norm:   1.8522087805273E+06
-   Residual 2-Norm:   1.8067743809550E-09
--- SOLUTION: Transport --
-   Time (seconds):   3.2598972320557E-03
-   Time Steps:           34
-   Newton Iterations:           41
-   Solver Iterations:           77
-   Time Step Cuts:            0
-   Solution 2-Norm:   9.9207029658907E-04
-   Residual 2-Norm:   1.4466808136692E-17

File regression_tests/default/543/543_flow_and_tracer_restart.chk30

Binary file removed.

File regression_tests/default/543/543_flow_and_tracer_restart.in

-:Description: 3D test problem for testing chemical reactions
-:             based on chemistry from Hammond and Lichtner, WRR, 2010
-
-:=========================== regression =======================================
-REGRESSION
-  CELLS_PER_PROCESS 2
-  CELLS
-    29
-  /
-END
-
-:=========================== flow mode ========================================
-MODE RICHARDS
-CHECKPOINT 30
-RESTART 543_flow_and_tracer_restart.chk30
-
-:=========================== chemistry ========================================
-CHEMISTRY
-  PRIMARY_SPECIES
-  Tracer
-  Tracer2
-  /
-  OUTPUT
-    All
-  /
-END
-
-:=========================== solver options ===================================
-TIMESTEPPER
-  TS_ACCELERATION 8
-END
-
-NEWTON_SOLVER FLOW
-END
-
-LINEAR_SOLVER FLOW
-  SOLVER DIRECT
-END
-
-NEWTON_SOLVER TRANSPORT
-END
-
-LINEAR_SOLVER TRANSPORT
-  SOLVER DIRECT
-END
-
-:=========================== discretization ===================================
-GRID
-  TYPE structured
-  ORIGIN 0.d0 0.d0 0.d0
-  NXYZ 5 4 3
-  DXYZ
-    10. 11. 12. 13. 14.
-    13. 12. 11. 10.
-    15. 20. 25.
-  /
-END
-
-:=========================== fluid properties =================================
-FLUID_PROPERTY 
-  DIFFUSION_COEFFICIENT 1.d-9
-END
-
-:=========================== datasets =========================================
-skip
-DATASET U
-  HDF5_DATASET_NAME U 
-  FILENAME datasets.h5
-END
-
-DATASET Calcite
-  HDF5_DATASET_NAME Calcite 
-  FILENAME datasets.h5
-END
-noskip
-
-:=========================== material properties ==============================
-MATERIAL_PROPERTY soil1
-  ID 1
-  POROSITY 0.25d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf1
-  PERMEABILITY
-    PERM_X 1.d-12
-    PERM_Y 1.d-12
-    PERM_Z 1.d-13
-  /
-END
-
-MATERIAL_PROPERTY soil2
-  ID 2
-  POROSITY 0.25d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf2
-  PERMEABILITY
-    PERM_X 1.d-11
-    PERM_Y 1.d-11
-    PERM_Z 1.d-12
-  /
-END
-
-MATERIAL_PROPERTY soil3
-  ID 3
-  POROSITY 0.2d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf3
-  PERMEABILITY
-    PERM_X 1.d-10
-    PERM_Y 1.d-10
-    PERM_Z 1.d-11
-  /
-END
-
-MATERIAL_PROPERTY soil4
-  ID 4
-  POROSITY 0.25d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf4
-  PERMEABILITY
-    PERM_X 1.d-9
-    PERM_Y 1.d-9
-    PERM_Z 1.d-10
-  /
-END
-
-:=========================== saturation functions =============================
-SATURATION_FUNCTION sf1
-  SATURATION_FUNCTION_TYPE VAN_GENUCHTEN
-  RESIDUAL_SATURATION 0.115
-  LAMBDA 0.286
-  ALPHA 1.9401d-4
-END
-
-SATURATION_FUNCTION sf2
-  SATURATION_FUNCTION_TYPE VAN_GENUCHTEN
-  RESIDUAL_SATURATION 0.110
-  LAMBDA 0.486
-  ALPHA 3.8801d-4
-END
-
-SATURATION_FUNCTION sf3
-  SATURATION_FUNCTION_TYPE VAN_GENUCHTEN
-  RESIDUAL_SATURATION 0.118
-  LAMBDA 0.541
-  ALPHA 1.0211d-4
-END
-
-SATURATION_FUNCTION sf4
-  SATURATION_FUNCTION_TYPE VAN_GENUCHTEN
-  RESIDUAL_SATURATION 0.143
-  LAMBDA 0.527
-  ALPHA 5.1054d-5
-END
-
-:=========================== output options ===================================
-OUTPUT
-:  TIMES d 0.001 0.01 0.1 1. 10. 
-:  PERIODIC TIME 1 d
-  FORMAT TECPLOT BLOCK
-  VELOCITIES
-END
-
-:=========================== times ============================================
-TIME
-  FINAL_TIME 100.d0 d
-  INITIAL_TIMESTEP_SIZE 1.d-6 d
-  MAXIMUM_TIMESTEP_SIZE 10.d0 d
-END
-
-:=========================== regions ==========================================
-REGION all
-  COORDINATES
-    0.d0 0.d0 0.d0
-    60.d0 46.d0 60.d0
-  /
-END
-
-REGION top
-  FACE TOP
-  COORDINATES
-    0.d0 0.d0 60.d0
-    60.d0 46.d0 60.d0
-  /
-END
-
-REGION bottom
-  FACE BOTTOM
-  COORDINATES
-    0.d0 0.d0 0.d0
-    60.d0 46.d0 0.d0
-  /
-END
-
-REGION west
-  FACE WEST
-  COORDINATES
-    0.d0 0.d0 0.d0
-    0.d0 46.d0 60.d0
-  /
-END
-
-REGION east
-  FACE EAST
-  COORDINATES
-    60.d0 0.d0 0.d0
-    60.d0 46.d0 60.d0
-  /
-END
-
-REGION north
-  FACE NORTH
-  COORDINATES
-    0.d0 46.d0 0.d0
-    60.d0 46.d0 60.d0
-  /
-END
-
-REGION south
-  FACE SOUTH
-  COORDINATES
-    0.d0 0.d0 0.d0
-    60.d0 0.d0 60.d0
-  /
-END
-
-REGION plume
-  BLOCK 2 3 2 3 2 3
-END
-
-REGION well
-  BLOCK 4 4 2 3 3 3 
-END
-
-:=========================== flow conditions ==================================
-FLOW_CONDITION initial
-  TYPE
-    PRESSURE hydrostatic
-  /
-  DATUM 0.d0 0.d0 35.d0
-  GRADIENT
-    PRESSURE -1.6666667d-2 0.d0 0.d0
-  /
-  PRESSURE 101325 ! Pa
-END
-
-FLOW_CONDITION east
-  TYPE
-    PRESSURE hydrostatic
-  /
-  INTERPOLATION LINEAR
-  DATUM LIST
-    TIME_UNITS d
-    0.d0 0.d0 0.d0 34.d0
-    10.d0 0.d0 0.d0 39.d0
-    50.d0 0.d0 0.d0 33.d0
-    100.d0 0.d0 0.d0 34.d0
-  /
-  PRESSURE 101325 ! Pa
-END
-
-FLOW_CONDITION recharge
-  TYPE
-    FLUX neumann
-  /
-  FLUX 5.d0 cm/yr
-END
-
-FLOW_CONDITION injection_well
-  TYPE
-    RATE scaled_volumetric_rate
-  /
-  RATE 1 m^3/hr
-END
-
-:=========================== transport conditions =============================
-TRANSPORT_CONDITION columbia_river
-  TYPE dirichlet_zero_gradient
-  CONSTRAINT_LIST
-    0.d0 river_water
-  /
-END
-
-TRANSPORT_CONDITION groundwater
-  TYPE dirichlet_zero_gradient
-  CONSTRAINT_LIST
-    0.d0 groundwater
-  /
-END
-
-TRANSPORT_CONDITION tracer_pulse
-  TYPE dirichlet
-  TIME_UNITS d
-  CONSTRAINT_LIST
-    0.d0 well_tracer
-    10.d0 groundwater
-  /
-END
-
-TRANSPORT_CONDITION rainwater
-  TYPE dirichlet
-  CONSTRAINT_LIST
-    0.d0 groundwater
-  /
-END
-
-TRANSPORT_CONDITION source_zone
-  TYPE dirichlet
-  CONSTRAINT_LIST
-    0.d0 U_source
-  /
-END
-
-:=========================== transport constraints ============================
-CONSTRAINT groundwater
-  CONCENTRATIONS
-    Tracer   1.e-7            F
-    Tracer2  1.e-7            F
-  /
-END
-
-CONSTRAINT U_source
-  CONCENTRATIONS
-    Tracer   1.e-7            F
-    Tracer2  1.e-7            F
-  /
-END
-
-CONSTRAINT river_water
-  CONCENTRATIONS
-    Tracer   1.e-3            F
-    Tracer2  1.e-7            F
-  /
-END
-
-CONSTRAINT well_tracer
-  CONCENTRATIONS
-    Tracer   1.e-7            F
-    Tracer2  1.e-3            F
-  /
-END
-
-:=========================== condition couplers ===============================
-: initial condition
-INITIAL_CONDITION
-  FLOW_CONDITION initial
-  TRANSPORT_CONDITION groundwater
-  REGION all
-END
-
-INITIAL_CONDITION
-  FLOW_CONDITION initial
-  TRANSPORT_CONDITION source_zone
-  REGION plume
-END
-
-: west boundary condition
-BOUNDARY_CONDITION west
-  FLOW_CONDITION initial
-  TRANSPORT_CONDITION groundwater
-  REGION west
-END
-
-: east boundary condition
-BOUNDARY_CONDITION east
-  FLOW_CONDITION east
-  TRANSPORT_CONDITION columbia_river
-  REGION east
-END
-
-: top boundary condition
-BOUNDARY_CONDITION top
-  FLOW_CONDITION recharge
-  TRANSPORT_CONDITION rainwater
-  REGION top
-END
-
-: well source/sink
-SOURCE_SINK well
-  FLOW_CONDITION injection_well
-  TRANSPORT_CONDITION tracer_pulse
-  REGION well
-END
-
-:=========================== stratigraphy couplers ============================
-STRATA
-  MATERIAL ./543.h5
-END
-

File regression_tests/default/543/543_flow_and_tracer_restart.regression.gold

--- PRESSURE: Liquid Pressure --
-      Max:   3.6989366949274E+05
-      Min:  -1.1821124686746E+04
-     Mean:   1.9217678931300E+05
-       29:   1.8951454146841E+05
-        1:   3.6935555583812E+05
-       31:   1.9891824328593E+05
--- SATURATION: Liquid Saturation --
-      Max:   1.0000000000000E+00
-      Min:   1.3540404806903E-01
-     Mean:   7.3892951211287E-01
-       29:   1.0000000000000E+00
-        1:   1.0000000000000E+00
-       31:   1.0000000000000E+00
--- CONCENTRATION: Total Tracer --
-      Max:   4.2831034034127E-04
-      Min:   1.0110785722184E-07
-     Mean:   6.8506830296352E-05
-       29:   1.1912361877853E-04
-        1:   1.0675826931978E-06
-       31:   5.3944919860991E-07
--- CONCENTRATION: Total Tracer2 --
-      Max:   6.9603781039616E-05
-      Min:   1.0001745985874E-07
-     Mean:   8.2896544604090E-06
-       29:   1.8697777128848E-07
-        1:   1.0003327243546E-07
-       31:   1.0005267136363E-07
--- DISCRETE: Material ID --
-      Max:         4
-      Min:         1
-     Mean:   2.3333333333333E+00
-       29:         3
-        1:         1
-       31:         3
--- GENERIC: LIQUID VELOCITY [m/d] --
-       29:   1.6066523961759E-01 -1.0572585640582E-01  2.6592648991018E-04
-        1:   1.8283368315718E-02 -1.0872639336429E-03 -7.9798244063844E-04
-       31:   3.1866628861244E-01  4.8193171366941E-02 -9.7826575411157E-04
--- SOLUTION: Flow --
-   Time (seconds):   2.3279190063477E-03
-   Time Steps:           34
-   Newton Iterations:           90
-   Solver Iterations:           90
-   Time Step Cuts:            0
-   Solution 2-Norm:   1.8522088053513E+06
-   Residual 2-Norm:   5.1817069195777E-10
--- SOLUTION: Transport --
-   Time (seconds):   1.2359619140625E-03
-   Time Steps:           34
-   Newton Iterations:           34
-   Solver Iterations:           34
-   Time Step Cuts:            0
-   Solution 2-Norm:   9.9207035061598E-04
-   Residual 2-Norm:   4.8567409875642E-19

File regression_tests/default/543/543_hanford_srfcplx_base.in

   MATERIAL ./543.h5
 END
 
+:=========================== checkpoint / restart =============================
+CHECKPOINT 10
+

File regression_tests/mtest/test_rt_unit.py

 
 import os
 import re
+import struct
 import subprocess
 import unittest
 
-from regression_tests import RegressionTest
+from regression_tests import RegressionTest, TestStatus
 
 
 class RegressionTest_SetTestData(unittest.TestCase):
                       self.check_performance, self.testlog)
         self.assertEqual(self.rt._stochastic_realizations, 3)
 
+    #--- restart timestep -----------------------------------------------------
+    def test_restart_timestep_default(self):
+        self.assertEqual(self.rt._pflotran_args, None)
+
+    def test_restart_timestep_default2(self):
+        self.rt.setup(self.criteria, self.test_data, self.timeout,
+                      self.check_performance, self.testlog)
+        self.assertEqual(self.rt._pflotran_args, None)
+
+    def test_restart_timestep_int(self):
+        self.test_data['restart_timestep'] = 10
+        self.rt.setup(self.criteria, self.test_data, self.timeout,
+                      self.check_performance, self.testlog)
+        self.assertEqual(self.rt._restart_timestep, 10)
+
+    def test_restart_timestep_junk(self):
+        self.test_data['restart_timestep'] = "foo bar baz"
+        self.assertRaises(ValueError, self.rt.setup,
+                          self.criteria, self.test_data, self.timeout,
+                          self.check_performance, self.testlog)
+
     #--- set_criteria ---------------------------------------------------------
     # verify that RegressionTest._set_criteria() is assigning test
     # criteria with the expected priority
         self.assertEqual(num_failures, 1)
 
     def test_compare_values_error(self):
-        """Failure when compare_values throws an exception. 
+        """Failure when compare_values throws an exception.
 
         type: pets will cause compare_values to throw an exception.
 
                           gold_value, gold_value)
 
 
+class RegressionTest_CleanupGeneratedFiles(unittest.TestCase):
+    """Test logic for pre-run file cleanup. cleanup should only touch
+    files generated by the current test.
+
+    """
+
+    def setUp(self):
+        self.testlog = open("dummy.testlog", 'w')
+        self.status = TestStatus()
+        self.rt = RegressionTest()
+        # test_data is the test section from a config file
+        test_data = {"name": "dummy_test_name"}
+        # timeout as specified by the command line arg
+        timeout = None
+        # check performance as specified by the command line arg
+        check_performance = False
+        # criteria is the default-test-criteria section from the config file
+        criteria = {}
+        self.rt.setup(criteria, test_data, timeout,
+                      check_performance, self.testlog)
+
+        # list of files that should be preserved (not caputured by generic rules)
+        self._save_suffixes = (".in", ".h5", ".txt", "-np8.in")
+        self._generate_files(self._save_suffixes)
+
+        # list of files that should always be moved
+        self._rename_suffixes = [".regression", ".out", ".stdout", ]
+
+    def tearDown(self):
+        self.testlog.close()
+        self._remove_files(self._save_suffixes)
+
+    def _generate_files(self, suffixes):
+        """
+        Generate a bunch of files.
+        """
+        data = "12345"
+        for suffix in suffixes:
+            name = "{0}{1}".format(self.rt.name(), suffix)
+            with open(name, "w") as tmpfile:
+                tmpfile.write(data)
+            name = "{0}-{1}{2}".format(self.rt._RESTART_PREFIX, self.rt.name(), suffix)
+            with open(name, "w") as tmpfile:
+                tmpfile.write(data)
+
+    def _remove_files(self, suffixes):
+        """Remove any remaining files
+        """
+        for suffix in suffixes:
+            name = "{0}{1}".format(self.rt.name(), suffix)
+            if os.path.isfile(name):
+                os.remove(name)
+            name = "{0}{1}.old".format(self.rt.name(), suffix)
+            if os.path.isfile(name):
+                os.remove(name)
+            name = "{0}-{1}{2}".format(self.rt._RESTART_PREFIX, self.rt.name(), suffix)
+            if os.path.isfile(name):
+                os.remove(name)
+            name = "{0}-{1}{2}.old".format(self.rt._RESTART_PREFIX, self.rt.name(), suffix)
+            if os.path.isfile(name):
+                os.remove(name)
+
+    def test_cleanup_generated_files_standard(self):
+        """test cleanup files for a standard run
+        """
+        self._generate_files(self._rename_suffixes)
+        self.rt._cleanup_generated_files()
+
+        for suffix in self._save_suffixes:
+            # verify the "save" files have been preserved
+            name = "{0}{1}".format(self.rt.name(), suffix)
+            self.assertTrue(os.path.isfile(name), name)
+
+        for suffix in self._rename_suffixes:
+            # verify the generated files have been moved to ".old"
+            name = "{0}{1}".format(self.rt.name(), suffix)
+            self.assertFalse(os.path.isfile(name), name)
+            self.assertTrue(os.path.isfile(name + ".old"), name)
+
+        self._remove_files(self._rename_suffixes)
+
+    def test_cleanup_generated_files_stochastic(self):
+        """test cleanup files for a stochastic run
+        """
+        self.rt._stochastic_realizations = 2
+        self._rename_suffixes.extend(["R1.regression", "R2.regression",
+                                      "R1.out", "R2.out"])
+        self._generate_files(self._rename_suffixes)
+        self.rt._cleanup_generated_files()
+
+        for suffix in self._save_suffixes:
+            # verify the "save" files have been preserved
+            name = "{0}{1}".format(self.rt.name(), suffix)
+            self.assertTrue(os.path.isfile(name), name)
+
+        for suffix in self._rename_suffixes:
+            # verify the generated files have been moved to ".old"
+            name = "{0}{1}".format(self.rt.name(), suffix)
+            self.assertFalse(os.path.isfile(name), name)
+            self.assertTrue(os.path.isfile(name + ".old"), name)
+            
+        self._remove_files(self._rename_suffixes)
+
+    def test_cleanup_generated_files_restart(self):
+        """test file cleanup for a restart run
+        """
+        self.rt._restart_timestep = 10
+        self._rename_suffixes.extend(["-restart.chk", "-5.chk", "-10.chk"])
+        self._generate_files(self._rename_suffixes)
+        self.rt._cleanup_generated_files()
+
+        for suffix in self._save_suffixes:
+            # verify the "save" files have been preserved
+            name = "{0}{1}".format(self.rt.name(), suffix)
+            self.assertTrue(os.path.isfile(name), name)
+
+        # temp input file has been moved
+        name = "{0}-{1}.in".format(self.rt._RESTART_PREFIX, self.rt.name())
+        self.assertFalse(os.path.isfile(name), name)
+        self.assertTrue(os.path.isfile(name + ".old"), name)
+        for suffix in self._rename_suffixes:
+            # verify the generated files (orig run) have been moved to ".old"
+            name = "{0}{1}".format(self.rt.name(), suffix)
+            self.assertFalse(os.path.isfile(name), name)
+            self.assertTrue(os.path.isfile(name + ".old"), name)
+            # verify the generated files (restart run) have been moved to ".old"
+            name = "{0}-{1}{2}".format(self.rt._RESTART_PREFIX, self.rt.name(), suffix)
+            self.assertFalse(os.path.isfile(name), name)
+            self.assertTrue(os.path.isfile(name + ".old"), name)
+
+        self._remove_files(self._rename_suffixes)
+
+
+class RegressionTest_CompareRestart(unittest.TestCase):
+    """Tests to verify comparison of restart files.
+
+    """
+
+    def setUp(self):
+        self.testlog = open("dummy.testlog", 'w')
+        self.status = TestStatus()
+        self.rt = RegressionTest()
+        # test_data is the test section from a config file
+        test_data = {"name": "dummy_test_name", }
+        # timeout as specified by the command line arg
+        timeout = None
+        # check performance as specified by the command line arg
+        check_performance = False
+        # criteria is the default-test-criteria section from the config file
+        criteria = {}
+        self.rt.setup(criteria, test_data, timeout,
+                      check_performance, self.testlog)
+
+    def tearDown(self):
+        self.testlog.close()
+
+    def test_get_hash_missing_file(self):
+        """Mark a test fail if restart file is missing.
+        """
+        tmp_filename = "tmp.file"
+        file_hash = self.rt._get_binary_restart_hash(tmp_filename, self.status, self.testlog)
+        self.assertEqual(self.status.fail, 1)
+
+    def test_get_hash(self):
+        """Get the hash of a file.
+        """
+        tmp_filename = "tmp.file"
+        with open(tmp_filename, 'w') as tmpfile:
+            tmpfile.write(str("This is a file."))
+        file_hash = self.rt._get_binary_restart_hash(tmp_filename, self.status, self.testlog)
+        self.assertEqual(file_hash, "679a60d9c581d48aa428a323879edc40a01e2a9d")
+        os.remove(tmp_filename)
+
+    def test_compare_hashes_same(self):
+        """status.fail not set when hashing indentical files.
+        """
+        data = struct.pack('<HHHH',
+                           0x0000, 0xFFFF, 0x0000, 0xFFFF)
+        tmp_filename_1 = "{0}-restart.chk".format(self.rt.name())
+        with open(tmp_filename_1, 'wb') as tmpfile_1:
+            tmpfile_1.write(data)
+
+        tmp_filename_2 = "tmp-restart-{0}-restart.chk".format(self.rt.name())
+        with open(tmp_filename_2, 'wb') as tmpfile_2:
+            tmpfile_2.write(data)
+
+        self.rt._check_restart(self.status, self.testlog)
+        self.assertEqual(self.status.fail, 0)
+
+        os.remove(tmp_filename_1)
+        os.remove(tmp_filename_2)
+
+    def test_compare_hashes_different(self):
+        """Fail when hashing files that differ by a single bit.
+        """
+        data = struct.pack('<HHHH',
+                            0x0000, 0xFFFF, 0x0000, 0xFFFF)
+        tmp_filename_1 = "{0}-restart.chk".format(self.rt.name())
+        with open(tmp_filename_1, 'wb') as tmpfile_1:
+            tmpfile_1.write(data)
+
+        data = struct.pack('<HHHH',
+                            0x0000, 0xFFFF, 0x0010, 0xFFFF)
+        tmp_filename_2 = "tmp-restart-{0}-restart.chk".format(self.rt.name())
+        with open(tmp_filename_2, 'wb') as tmpfile_2:
+            tmpfile_2.write(data)
+
+        self.rt._check_restart(self.status, self.testlog)
+        self.assertEqual(self.status.fail, 1)
+
+        os.remove(tmp_filename_1)
+        os.remove(tmp_filename_2)
+
+
 if __name__ == '__main__':
     #unittest.main(buffer=True)
     unittest.main()

File regression_tests/regression_tests.py

 #!/bin/env python
 """
-Program to manage and run PFloTran regression tests
+Program to manage and run PFloTran regression tests.
+
 """
 
 from __future__ import print_function
 
 import argparse
 import datetime
+import hashlib
 import os
 import pprint
 import re
         self._TOL_VALUE = 0
         self._TOL_TYPE = 1
         self._PFLOTRAN_SUCCESS = 86
+        self._RESTART_PREFIX = "tmp-restart"
         # misc test parameters
         self._pprint = pprint.PrettyPrinter(indent=2)
         self._txtwrap = textwrap.TextWrapper(width=78, subsequent_indent=4*" ")
         self._np = None
         self._pflotran_args = None
         self._stochastic_realizations = None
+        self._restart_timestep = None
         self._timeout = 60.0
         self._check_performance = False
         self._num_failed = 0
         return self._test_name
 
     def run(self, mpiexec, executable, dry_run, status, testlog):
+        """Run the test.
+
+        If this is a restart test, then we will copy the input file,
+        append the restart flag and run the test a second time.
+
+        """
+        self._cleanup_generated_files()
+
+        self._run_test(mpiexec, executable, self.name(), dry_run, status,
+                       testlog)
+
+        if self._restart_timestep is not None:
+            restart_file = "{0}-{1}.chk".format(self.name(),
+                                                self._restart_timestep)
+            if os.path.isfile(restart_file):
+                restart_name = "{0}-{1}".format(self._RESTART_PREFIX,
+                                                self.name())
+                shutil.copy("{0}.in".format(self.name()),
+                            "{0}.in".format(restart_name))
+                with open("{0}.in".format(restart_name), 'a') as tempfile:
+                    tempfile.write("RESTART {0}\n".format(restart_file))
+                self._run_test(mpiexec, executable, restart_name, dry_run,
+                               status, testlog)
+            elif not status.skipped:
+                status.fail = 1
+                message = self._txtwrap.fill(
+                    "ERROR: restart test '{0}' did not generate a checkpoint "
+                    "file at the specified step: '{1}' ('{2}'). This can "
+                    "occur if the checkpoint interval and restart step are "
+                    "not consistent or the run failed.".format(
+                        self.name(), self._restart_timestep, restart_file))
+                print("".join(['\n', message, '\n']), file=testlog)
+
+    def _run_test(self, mpiexec, executable, test_name, dry_run, status, testlog):
         """
         Build up the run command, including mpiexec, np, pflotran,
         input file, output file. Then run the job as a subprocess.
         command.append("0")
         if self._input_arg is not None:
             command.append(self._input_arg)
-            command.append(self.name())
+            command.append(test_name)
 
         if self._pflotran_args is not None:
             # assume that we already called split() on the
             # may be empty)
             command = command + self._pflotran_args
 
-        if os.path.isfile(self.name() + ".regression"):
-            os.rename(self.name() + ".regression",
-                      self.name() + ".regression.old")
-
-        if os.path.isfile(self.name() + ".out"):
-            os.rename(self.name() + ".out",
-                      self.name() + ".out.old")
-
-        if os.path.isfile(self.name() + ".stdout"):
-            os.rename(self.name() + ".stdout",
-                      self.name() + ".stdout.old")
-
         if not dry_run:
             print("    cd {0}".format(os.getcwd()), file=testlog)
             print("    {0}".format(" ".join(command)), file=testlog)
-            run_stdout = open(self.name() + ".stdout", 'w')
+            run_stdout = open(test_name + ".stdout", 'w')
             start = time.time()
             proc = subprocess.Popen(command,
                                     shell=False,
                     time.sleep(0.1)
                     message = self._txtwrap.fill(
                         "ERROR: job '{0}' has exceeded timeout limit of "
-                        "{1} seconds.".format(self.name(), self._timeout))
+                        "{1} seconds.".format(test_name, self._timeout))
                     print(''.join(['\n', message, '\n']), file=testlog)
             pflotran_status = abs(proc.returncode)
             run_stdout.close()
                 "code ({status}) indicating the simulation may have "
                 "failed. Please check '{name}.out' and '{name}.stdout' "
                 "for error messages (included below).".format(
-                    name=self.name(), status=pflotran_status))
+                    name=test_name, status=pflotran_status))
             print("".join(['\n', message, '\n']), file=testlog)
-            print("~~~~~ {0}.stdout ~~~~~".format(self.name()), file=testlog)
-            with open("{0}.stdout".format(self.name()), 'r') as tempfile:
+            print("~~~~~ {0}.stdout ~~~~~".format(test_name), file=testlog)
+            with open("{0}.stdout".format(test_name), 'r') as tempfile:
                 shutil.copyfileobj(tempfile, testlog)
-            print("~~~~~ {0}.out ~~~~~".format(self.name()), file=testlog)
-            with open("{0}.out".format(self.name()), 'r') as tempfile:
+            print("~~~~~ {0}.out ~~~~~".format(test_name), file=testlog)
+            with open("{0}.out".format(test_name), 'r') as tempfile:
                 shutil.copyfileobj(tempfile, testlog)
             print("~~~~~~~~~~", file=testlog)
 
+    def _cleanup_generated_files(self):
+        """Cleanup old generated files that may be hanging around from a
+        previous run by renaming them with .old appended to the name.
+
+        NOTE:
+
+          - Do NOT match files with something like "if self.name() in
+            filename" or "if filenamename.startswith(self.name())"
+            because this will capture files from another test with a
+            similar name, e.g.  "test_flow" and "test_flow_np4" would
+            both be captured.
+
+          - This assumes that all files with the listed suffixes are
+            old. That means checkpoint files must be generated, not
+            saved.
+
+        """
+        # files from a normal run
+        suffixes = ["regression", "out", "stdout"]
+        for suffix in suffixes:
+            name = "{0}.{1}".format(self.name(), suffix)
+            if os.path.isfile(name):
+                os.rename(name, name + ".old")
+
+        # files from a stochastic run
+        if self._stochastic_realizations is not None:
+            for i in range(1, self._stochastic_realizations + 1):
+                run_id = "R{0}".format(i)
+                for suffix in suffixes:
+                    name = "{0}{1}.{2}".format(self.name(), run_id, suffix)
+                    if os.path.isfile(name):
+                        os.rename(name, name + ".old")
+
+        # temp files from a restart run
+        if self._restart_timestep is not None:
+            suffixes.append("in")
+            for suffix in suffixes:
+                name = "{0}-{1}.{2}".format(self._RESTART_PREFIX, self.name(), suffix)
+                if os.path.isfile(name):
+                    os.rename(name, name + ".old")
+
+            # checkpoint/restart files, both from the original and restart run
+            cwd = os.getcwd()
+            for entry in os.listdir(cwd):
+                if os.path.isfile(entry):
+                    search_checkpoint = "^({0}-)?{1}-([\d]+|restart).chk$".format(
+                        self._RESTART_PREFIX, self.name())
+                    if re.search(search_checkpoint, entry):
+                        os.rename(entry, entry + ".old")
+
     def check(self, status, testlog):
         """
         Check the test results against the gold standard
         else:
             self._check_gold(status, run_id, testlog)
 
+        if self._restart_timestep is not None:
+            self._check_restart(status, testlog)
+
     def _check_gold(self, status, run_id, testlog):
         """
         Test the output from the run against the known "gold standard"
         if self._num_failed > 0:
             status.fail = 1
 
+    def _check_restart(self, status, testlog):
+        """Check that binary restart files are bit for bit after a restart.
+
+        The entire restart file should be bit for bit, both the "big"
+        data and the metadata. This means we can take a hash of the
+        file and report even a single bit difference between the files
+        as a failure. If the meta data is allowd to be different, we
+        will need a more sophisticated way of diffing the files.
+
+        """
+
+        orig_filename="{0}-restart.chk".format(self.name())
+        restart_filename="{0}-{1}".format(self._RESTART_PREFIX, orig_filename)
+
+        print("\n    comparing restart files:\n        {0}\n        {1}".format(
+            orig_filename, restart_filename), file=testlog)
+
+        orig_hash = self._get_binary_restart_hash(orig_filename,
+                                                  status, testlog)
+        restart_hash = self._get_binary_restart_hash(restart_filename,
+                                                     status, testlog)
+
+        if orig_hash is not False and restart_hash is not False: 
+            if orig_hash != restart_hash:
+                print("    FAIL: final restart files are not bit for bit "
+                      "identical.", file=testlog)
+                status.fail = 1
+            else:
+                print("    bit for bit restart test passed.\n", file=testlog)
+
+    def _get_binary_restart_hash(self, filename, status, testlog):
+        """Get the sha1 hash of a restart file. The hash should be different
+        if even one bit is different in the file.
+
+        """
+        hash_digest = False
+        if os.path.isfile(filename):
+            restart_hash = hashlib.sha1()
+            with open(filename, mode='rb') as restart_file:
+                restart_hash.update(restart_file.read())
+            hash_digest = restart_hash.hexdigest()
+        else:
+            message = self._txtwrap.fill(
+                "FAIL: could not find restart file '{0}'".format(filename))
+            print("".join(['\n', message, '\n']), file=testlog)
+            status.fail = 1
+        return hash_digest
+
     def update(self, status, testlog):
         """
         Update the gold standard test results to the current
                         "num_realizations flag as well. "
                         "test : {0}".format(self.name()))
 
+        self._restart_timestep = test_data.pop('restart_timestep', None)
+        if self._restart_timestep is not None:
+            try:
+                self._restart_timestep = int(self._restart_timestep)
+            except ValueError as error:
+                raise ValueError("ERROR: restart_timestep must be an integer value. "
+                                "test : {0}".format(self.name()))
+
         self._check_performance = check_performance
 
         # timeout : preference (1) command line (2) test data (3) class default

File regression_tests/shortcourse/1D_Calcite/calcite.cfg

 concentration = 1.0e-9 absolute
 
 [calcite_vsat_flow_and_tran]
+#restart_timestep = 25
 pressure = 1.0e-6 absolute
 

File regression_tests/shortcourse/1D_Calcite/calcite_vsat_flow_and_tran.in

   MATERIAL soil1
 END
 
+:=========================== checkpoint / restart =============================
+CHECKPOINT 25
 

File regression_tests_refactor/Makefile

 	-find . -type f -name '*.stdout' -print0 | xargs -0 rm
 	-find . -type f -name '*.old' -print0 | xargs -0 rm
 	-find . -type f -name '*~' -print0 | xargs -0 rm
+	-find . -type f -name 'tmp-restart-*' -print0 | xargs -0 rm
+	-find . -type f -name '*.chk' -print0 | xargs -0 rm
 	-rm -f default/anisothermal/thc_1d.h5
 	-rm -f default/anisothermal/th_1d.h5
 	-rm -f mfd/mfd-mas.dat
 	-rm -f mfd/mfd.h5
-	-rm -f default/543/543_flow_and_tracer_restart-np8-restart.chk
-	-rm -f default/543/543_flow_and_tracer_restart-restart.chk

File regression_tests_refactor/default/543/543.cfg

 [suites]
 flow = 543_flow
-transport = 543_flow_and_tracer 543_flow_and_tracer_restart 543_flow_and_tracer_init_to_ss 543_flow_and_tracer_run_as_ss
+transport = 543_flow_and_tracer 543_flow_and_tracer_init_to_ss 543_flow_and_tracer_run_as_ss
 geochemistry = 543_hanford_srfcplx_base 543_hanford_srfcplx_param 543_hanford_srfcplx_mr
-parallel = 543_flow-np8 543_flow_and_tracer-np8 543_hanford_srfcplx_param-np8 543_flow_and_tracer_restart-np8
+parallel = 543_flow-np8 543_flow_and_tracer-np8 543_hanford_srfcplx_param-np8
 
 standard = 543_flow 
            543_flow_and_tracer
-           543_flow_and_tracer_restart
            543_hanford_srfcplx_base
            543_hanford_srfcplx_param 
            543_hanford_srfcplx_mr
 #           543_hanford_srfcplx_mr
 standard_parallel = 543_flow-np8
                     543_flow_and_tracer-np8
-                    543_flow_and_tracer_restart-np8
                     543_hanford_srfcplx_param-np8
 
 [default-test-criteria]
 np=8
 
 [543_flow_and_tracer]
+#restart_timestep = 10
 
 [543_flow_and_tracer-np8]
 np=8
+#restart_timestep = 10
 
 [543_hanford_srfcplx_base]
+#restart_timestep = 10
 
 [543_hanford_srfcplx_mr]
 
 [543_hanford_srfcplx_param-np8]
 np=8
 
-[543_flow_and_tracer_restart]
-
-[543_flow_and_tracer_restart-np8]
-np=8
-
 [543_flow_and_tracer_init_to_ss]
 
 [543_flow_and_tracer_run_as_ss]

File regression_tests_refactor/default/543/543_flow_and_tracer-np8.in

   MATERIAL ./543.h5
 END
 
+:=========================== checkpoint / restart =============================
+CHECKPOINT 10
+

File regression_tests_refactor/default/543/543_flow_and_tracer.in

   MATERIAL ./543.h5
 END
 
+:=========================== checkpoint / restart =============================
+CHECKPOINT 10
+

File regression_tests_refactor/default/543/543_flow_and_tracer_restart-np8.chk30

Binary file removed.

File regression_tests_refactor/default/543/543_flow_and_tracer_restart-np8.in

-:Description: 3D test problem for testing chemical reactions
-:             based on chemistry from Hammond and Lichtner, WRR, 2010
-
-:=========================== regression =======================================
-REGRESSION
-  CELLS_PER_PROCESS 2
-  CELLS
-    29
-  /
-END
-
-:=========================== flow mode ========================================
-MODE RICHARDS
-CHECKPOINT 30
-RESTART 543_flow_and_tracer_restart-np8.chk30
-
-:=========================== chemistry ========================================
-CHEMISTRY
-  PRIMARY_SPECIES
-  Tracer
-  Tracer2
-  /
-  OUTPUT
-    All
-  /
-END
-
-:=========================== solver options ===================================
-TIMESTEPPER
-  TS_ACCELERATION 8
-END
-
-NEWTON_SOLVER FLOW
-END
-
-LINEAR_SOLVER FLOW
-END
-
-NEWTON_SOLVER TRANSPORT
-END
-
-LINEAR_SOLVER TRANSPORT
-END
-
-:=========================== discretization ===================================
-GRID
-  TYPE structured
-  ORIGIN 0.d0 0.d0 0.d0
-  NXYZ 5 4 3
-  DXYZ
-    10. 11. 12. 13. 14.
-    13. 12. 11. 10.
-    15. 20. 25.
-  /
-END
-
-:=========================== fluid properties =================================
-FLUID_PROPERTY 
-  DIFFUSION_COEFFICIENT 1.d-9
-END
-
-:=========================== datasets =========================================
-skip
-DATASET U
-  HDF5_DATASET_NAME U 
-  FILENAME datasets.h5
-END
-
-DATASET Calcite
-  HDF5_DATASET_NAME Calcite 
-  FILENAME datasets.h5
-END
-noskip
-
-:=========================== material properties ==============================
-MATERIAL_PROPERTY soil1
-  ID 1
-  POROSITY 0.25d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf1
-  PERMEABILITY
-    PERM_X 1.d-12
-    PERM_Y 1.d-12
-    PERM_Z 1.d-13
-  /
-END
-
-MATERIAL_PROPERTY soil2
-  ID 2
-  POROSITY 0.25d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf2
-  PERMEABILITY
-    PERM_X 1.d-11
-    PERM_Y 1.d-11
-    PERM_Z 1.d-12
-  /
-END
-
-MATERIAL_PROPERTY soil3
-  ID 3
-  POROSITY 0.2d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf3
-  PERMEABILITY
-    PERM_X 1.d-10
-    PERM_Y 1.d-10
-    PERM_Z 1.d-11
-  /
-END
-
-MATERIAL_PROPERTY soil4
-  ID 4
-  POROSITY 0.25d0
-  TORTUOSITY 1.d0
-  SATURATION_FUNCTION sf4
-  PERMEABILITY