[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::Creating simulation cptest
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::Simulation directory: /lustre/AEI/ianhin/simulations/cptest
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::Simulation Properties:
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::[properties]
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::finished        = no
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::machine         = datura
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::simulationid    = simulation-cptest-datura-login-damiana.aei.mpg.de-ianhin-2011.02.27-17.36.12-18857
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::sourcedir       = /home/ianhin/Cactus/llama
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::configuration   = sim2_datura
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::configid        = config-sim2_datura-login-damiana.damiana.admin-home-ianhin-Cactus-llama
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::buildid         = build-sim2_datura-login-damiana.damiana.admin-ianhin-2011.02.25-10.16.22-18331
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::executable      = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/exe/cactus_sim2_datura
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::optionlist      = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/cfg/OptionList
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::submitscript    = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/SubmitScript
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::runscript       = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/RunScript
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::parfile         = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/par/cptest.par
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::
[LOG:2011-02-27 17:36:12] restart.create(simulationName, parfile)::Simulation cptest created
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::Restart for simulation cptest created with restart id 0, long restart id 0000
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::Prepping for submission
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::No previous walltime available to be used, using walltime 1:00:00
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::Defined substituion properties for submission
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::{'SIMULATION_ID': 'simulation-cptest-datura-login-damiana.aei.mpg.de-ianhin-2011.02.27-17.36.12-18857', 'PPN_USED': 12, 'PPN': 12, 'QUEUE': 'daturamon.q', 'WALLTIME_HH': '01', 'CPUFREQ': '2666', 'USER': 'ianhin', 'RUNDIR': '/lustre/AEI/ianhin/simulations/cptest/output-0000', 'NODES': 1, 'SIMULATION_NAME': 'cptest', 'WALLTIME': '1:00:00', 'NUM_THREADS': 1, 'EXECUTABLE': '/lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/exe/cactus_sim2_datura', 'PROCS_REQUESTED': 12, 'EMAIL': 'ian.hinder@aei.mpg.de', 'RESTART_ID': 0, 'CHAINED_JOB_ID': '', 'FROM_RESTART_COMMAND': '', 'WALLTIME_SECONDS': 3600, 'SIMFACTORY': '/home/ianhin/Cactus/llama/simfactory2/bin/sim', 'PROCS': 12, 'SUBMITSCRIPT': '/lustre/AEI/ianhin/simulations/cptest/output-0000/SIMFACTORY/SubmitScript', 'WALLTIME_HOURS': 1.0, 'WALLTIME_MM': '00', 'PARFILE': '/lustre/AEI/ianhin/simulations/cptest/output-0000/cptest.par', 'WALLTIME_SS': '00', 'ALLOCATION': 'TG-MCA02N014', 'SOURCEDIR': '/home/ianhin/Cactus/llama', 'HOSTNAME': 'login-damiana.aei.mpg.de', 'NUM_PROCS': 12, 'SCRIPTFILE': '/lustre/AEI/ianhin/simulations/cptest/output-0000/SIMFACTORY/SubmitScript', 'MEMORY': '24576', 'WALLTIME_MINUTES': 60, 'SHORT_SIMULATION_NAME': 'cptest-0000'}
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::self.Properties: /lustre/AEI/ianhin/simulations/cptest/output-0000/SIMFACTORY/properties.ini
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::[properties]
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::finished        = no
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::machine         = datura
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::simulationid    = simulation-cptest-datura-login-damiana.aei.mpg.de-ianhin-2011.02.27-17.36.12-18857
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::sourcedir       = /home/ianhin/Cactus/llama
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::configuration   = sim2_datura
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::configid        = config-sim2_datura-login-damiana.damiana.admin-home-ianhin-Cactus-llama
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::buildid         = build-sim2_datura-login-damiana.damiana.admin-ianhin-2011.02.25-10.16.22-18331
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::executable      = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/exe/cactus_sim2_datura
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::optionlist      = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/cfg/OptionList
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::submitscript    = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/SubmitScript
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::runscript       = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/RunScript
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::parfile         = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/par/cptest.par
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::chainedjobid    = -1
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::ppn             = 12
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::procsrequested  = 12
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::allocation      = TG-MCA02N014
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::user            = ianhin
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::procs           = 12
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::walltime        = 1:00:00
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::numprocs        = 12
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::numthreads      = 1
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::hostname        = login-damiana.aei.mpg.de
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::ppnused         = 12
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::queue           = daturamon.q
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::cpufreq         = 2666
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::memory          = 24576
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::nodes           = 1
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::pbsSimulationName= cptest-0000
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::saving substituted submitscript contents to: /lustre/AEI/ianhin/simulations/cptest/output-0000/SIMFACTORY/SubmitScript
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::Executing submission command: qsub /lustre/AEI/ianhin/simulations/cptest/output-0000/SIMFACTORY/SubmitScript
[LOG:2011-02-27 17:36:12] self.makeActive()::Simulation cptest with restart-id 0 has been made active
[LOG:2011-02-27 17:36:12] job_id = self.extractJobId(output)::received raw output: Your job 255257 ("cptest-0000") has been submitted
[LOG:2011-02-27 17:36:12] job_id = self.extractJobId(output)::
[LOG:2011-02-27 17:36:12] job_id = self.extractJobId(output)::using submitRegex: (\d+)
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::After searching raw output, it was determined that the job_id is: 255257
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::If this is -1, that means the regex did NOT match anything. No job_id means no control.
[LOG:2011-02-27 17:36:12] self.submit(submitScript)::Simulation cptest, with restart id 0, and job id 255257 has been submitted
[LOG:2011-02-27 17:36:16] restart.submitRun(simulationName, restart_id)::submitRun
[LOG:2011-02-27 17:36:16] self.load(simulationName, restartId)::For simulation cptest, loaded restart id 0, long restart id 0000
[LOG:2011-02-27 17:36:16] restart.submitRun(simulationName, restart_id)::No previous restart to clean up
[LOG:2011-02-27 17:36:16] restart.submitRun(simulationName, restart_id)::submitRun/run
[LOG:2011-02-27 17:36:16] self.run()::Prepping for execution/run
[LOG:2011-02-27 17:36:16] checkpointing = self.PrepareCheckpointing(self.MaxRestartID)::PrepareCheckpointing: max_restart_id: 0
[LOG:2011-02-27 17:36:16] checkpointing = self.PrepareCheckpointing(self.MaxRestartID)::max_restart_id == self.RestartID, returning false
[LOG:2011-02-27 17:36:16] self.GenerateNodefile()::mpi_nodefile did not exist, so we attempted to make it. The contents are below
[LOG:2011-02-27 17:36:16] self.GenerateNodefile()::kop30.datura.admin 12 daturamon.q@kop30.datura.admin <NULL>
[LOG:2011-02-27 17:36:16] self.GenerateNodefile()::
[LOG:2011-02-27 17:36:16] self.run()::Defined substituion properties for execution/run
[LOG:2011-02-27 17:36:16] self.run()::{'SIMULATION_ID': 'simulation-cptest-datura-login-damiana.aei.mpg.de-ianhin-2011.02.27-17.36.12-18857', 'PPN_USED': '12', 'PPN': '12', 'WALLTIME_HH': '01', 'CPUFREQ': '2666', 'USER': 'ianhin', 'RUNDIR': '/lustre/AEI/ianhin/simulations/cptest/output-0000', 'NODES': '1', 'SIMULATION_NAME': 'cptest', 'WALLTIME': '1:00:00', 'NUM_THREADS': '1', 'EXECUTABLE': '/lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/exe/cactus_sim2_datura', 'SCRATCHDIR': '/tmp/${USER}/${JOB_ID}', 'PROCS_REQUESTED': '12', 'RESTART_ID': 0, 'WALLTIME_SECONDS': 3600, 'PROCS': '12', 'SUBMITSCRIPT': '/lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/SubmitScript', 'WALLTIME_MM': '00', 'PARFILE': '/lustre/AEI/ianhin/simulations/cptest/output-0000/cptest.par', 'WALLTIME_SS': '00', 'WALLTIME_HOURS': 1.0, 'SOURCEDIR': '/home/ianhin/Cactus/llama', 'HOSTNAME': 'login-damiana.aei.mpg.de', 'RUNDEBUG': 0, 'NUM_PROCS': '12', 'SCRIPTFILE': '/lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/SubmitScript', 'MEMORY': '24576', 'WALLTIME_MINUTES': 60, 'SHORT_SIMULATION_NAME': 'cptest-0000'}
[LOG:2011-02-27 17:36:16] self.run()::Executing run command: /lustre/AEI/ianhin/simulations/cptest/output-0000/SIMFACTORY/RunScript
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::Restart for simulation cptest created with restart id 1, long restart id 0001
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::Prepping for submission
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::Using walltime 1:00:00 from previous restart 0
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::Defined substituion properties for submission
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::{'SIMULATION_ID': 'simulation-cptest-datura-login-damiana.aei.mpg.de-ianhin-2011.02.27-17.36.12-18857', 'PPN_USED': 12, 'PPN': 12, 'QUEUE': 'daturamon.q', 'WALLTIME_HH': '01', 'CPUFREQ': '2666', 'USER': 'ianhin', 'RUNDIR': '/lustre/AEI/ianhin/simulations/cptest/output-0001', 'NODES': 1, 'SIMULATION_NAME': 'cptest', 'WALLTIME': '1:00:00', 'NUM_THREADS': 1, 'EXECUTABLE': '/lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/exe/cactus_sim2_datura', 'PROCS_REQUESTED': 12, 'EMAIL': 'ian.hinder@aei.mpg.de', 'RESTART_ID': 1, 'CHAINED_JOB_ID': '', 'FROM_RESTART_COMMAND': '', 'WALLTIME_SECONDS': 3600, 'SIMFACTORY': '/home/ianhin/Cactus/llama/simfactory2/bin/sim', 'PROCS': 12, 'SUBMITSCRIPT': '/lustre/AEI/ianhin/simulations/cptest/output-0001/SIMFACTORY/SubmitScript', 'WALLTIME_HOURS': 1.0, 'WALLTIME_MM': '00', 'PARFILE': '/lustre/AEI/ianhin/simulations/cptest/output-0001/cptest.par', 'WALLTIME_SS': '00', 'ALLOCATION': 'TG-MCA02N014', 'SOURCEDIR': '/home/ianhin/Cactus/llama', 'HOSTNAME': 'login-damiana.aei.mpg.de', 'NUM_PROCS': 12, 'SCRIPTFILE': '/lustre/AEI/ianhin/simulations/cptest/output-0001/SIMFACTORY/SubmitScript', 'MEMORY': '24576', 'WALLTIME_MINUTES': 60, 'SHORT_SIMULATION_NAME': 'cptest-0001'}
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::self.Properties: /lustre/AEI/ianhin/simulations/cptest/output-0001/SIMFACTORY/properties.ini
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::[properties]
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::finished        = no
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::machine         = datura
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::simulationid    = simulation-cptest-datura-login-damiana.aei.mpg.de-ianhin-2011.02.27-17.36.12-18857
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::sourcedir       = /home/ianhin/Cactus/llama
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::configuration   = sim2_datura
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::configid        = config-sim2_datura-login-damiana.damiana.admin-home-ianhin-Cactus-llama
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::buildid         = build-sim2_datura-login-damiana.damiana.admin-ianhin-2011.02.25-10.16.22-18331
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::executable      = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/exe/cactus_sim2_datura
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::optionlist      = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/cfg/OptionList
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::submitscript    = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/SubmitScript
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::runscript       = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/RunScript
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::parfile         = /lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/par/cptest.par
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::chainedjobid    = -1
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::ppn             = 12
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::procsrequested  = 12
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::allocation      = TG-MCA02N014
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::user            = ianhin
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::procs           = 12
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::walltime        = 1:00:00
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::numprocs        = 12
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::numthreads      = 1
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::hostname        = login-damiana.aei.mpg.de
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::ppnused         = 12
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::queue           = daturamon.q
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::cpufreq         = 2666
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::memory          = 24576
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::nodes           = 1
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::pbsSimulationName= cptest-0001
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::saving substituted submitscript contents to: /lustre/AEI/ianhin/simulations/cptest/output-0001/SIMFACTORY/SubmitScript
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::Executing submission command: qsub /lustre/AEI/ianhin/simulations/cptest/output-0001/SIMFACTORY/SubmitScript
[LOG:2011-02-27 17:39:07] self.makeActive()::Simulation cptest with restart-id 1 has been made active
[LOG:2011-02-27 17:39:07] job_id = self.extractJobId(output)::received raw output: Your job 255258 ("cptest-0001") has been submitted
[LOG:2011-02-27 17:39:07] job_id = self.extractJobId(output)::
[LOG:2011-02-27 17:39:07] job_id = self.extractJobId(output)::using submitRegex: (\d+)
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::After searching raw output, it was determined that the job_id is: 255258
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::If this is -1, that means the regex did NOT match anything. No job_id means no control.
[LOG:2011-02-27 17:39:07] self.submit(submitScript)::Simulation cptest, with restart id 1, and job id 255258 has been submitted
[LOG:2011-02-27 17:39:16] restart.submitRun(simulationName, restart_id)::submitRun
[LOG:2011-02-27 17:39:16] self.load(simulationName, restartId)::For simulation cptest, loaded restart id 1, long restart id 0001
[LOG:2011-02-27 17:39:16] restart.submitRun(simulationName, restart_id)::No previous restart to clean up
[LOG:2011-02-27 17:39:16] restart.submitRun(simulationName, restart_id)::submitRun/run
[LOG:2011-02-27 17:39:16] self.run()::Prepping for execution/run
[LOG:2011-02-27 17:39:16] checkpointing = self.PrepareCheckpointing(self.MaxRestartID)::PrepareCheckpointing: max_restart_id: 1
[LOG:2011-02-27 17:39:16] checkpointing = self.PrepareCheckpointing(self.MaxRestartID)::max_restart_id == self.RestartID, returning false
[LOG:2011-02-27 17:39:16] self.GenerateNodefile()::mpi_nodefile did not exist, so we attempted to make it. The contents are below
[LOG:2011-02-27 17:39:16] self.GenerateNodefile()::kop184.datura.admin 12 daturamon.q@kop184.datura.admin <NULL>
[LOG:2011-02-27 17:39:16] self.GenerateNodefile()::
[LOG:2011-02-27 17:39:16] self.run()::Defined substituion properties for execution/run
[LOG:2011-02-27 17:39:16] self.run()::{'SIMULATION_ID': 'simulation-cptest-datura-login-damiana.aei.mpg.de-ianhin-2011.02.27-17.36.12-18857', 'PPN_USED': '12', 'PPN': '12', 'WALLTIME_HH': '01', 'CPUFREQ': '2666', 'USER': 'ianhin', 'RUNDIR': '/lustre/AEI/ianhin/simulations/cptest/output-0001', 'NODES': '1', 'SIMULATION_NAME': 'cptest', 'WALLTIME': '1:00:00', 'NUM_THREADS': '1', 'EXECUTABLE': '/lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/exe/cactus_sim2_datura', 'SCRATCHDIR': '/tmp/${USER}/${JOB_ID}', 'PROCS_REQUESTED': '12', 'RESTART_ID': 1, 'WALLTIME_SECONDS': 3600, 'PROCS': '12', 'SUBMITSCRIPT': '/lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/SubmitScript', 'WALLTIME_MM': '00', 'PARFILE': '/lustre/AEI/ianhin/simulations/cptest/output-0001/cptest.par', 'WALLTIME_SS': '00', 'WALLTIME_HOURS': 1.0, 'SOURCEDIR': '/home/ianhin/Cactus/llama', 'HOSTNAME': 'login-damiana.aei.mpg.de', 'RUNDEBUG': 0, 'NUM_PROCS': '12', 'SCRIPTFILE': '/lustre/AEI/ianhin/simulations/cptest/SIMFACTORY/run/SubmitScript', 'MEMORY': '24576', 'WALLTIME_MINUTES': 60, 'SHORT_SIMULATION_NAME': 'cptest-0001'}
[LOG:2011-02-27 17:39:16] self.run()::Executing run command: /lustre/AEI/ianhin/simulations/cptest/output-0001/SIMFACTORY/RunScript
