VASP (scorep)

[jg piccinali]

scorep/1.4.2

daint

Setup

• module swap PrgEnv-cray PrgEnv-intel
• module load craype-accel-nvidia35
• module load fftw
• # module load magma # cf makefile.include
• /apps/santis/sandbox/jgp/vasp/src/541

Compile

• mkdir build
• vim src/CUDA/common.mk
• vim makefile.include
• make gpu -j2

Run

• cd /apps/santis/sandbox/jgp/vasp/src/RUN/CeO2
• export CRAY_CUDA_MPS=1
• sbatch.sh santis 10 ../../541/bin/vasp_gpu 16 8 1 # real 439.44

Compile (scorep)

• export PATH=/apps/daint/5.2.UP02/scorep/1.4.2/int1501sci1304mpi722cuda6514acc311otf151opa114cube431/bin:$PATH
• vim src/CUDA/common.mk
• vim makefile.include

Profiling (scorep)

• export LD_LIBRARY_PATH=/apps/daint/5.2.UP02/magma/1.6.2/intel_1402/lib/:$LD_LIBRARY_PATH
• export SCOREP_ENABLE_PROFILING=true
• export SCOREP_ENABLE_TRACING=false
• export SCOREP_CUDA_ENABLE=yes
• export CRAY_CUDA_MPS=1
• sbatch.sh santis 10 vasp_gpu+sc142 16 8 1

# [Score-P] src/adapters/cuda/scorep_cupti_activity.c:557: 
Warning: [CUPTI Activity] Memcpy: start time < 
last written timestamp! (CUDA device:stream [0:7])
# [Score-P] src/adapters/cuda/scorep_cupti_activity.c:564: 
Warning: [CUPTI Activity] Set memcpy start time
to sync-point time (truncate 0.1725%)

• sbatch.sh santis 10 ../../541/bin/vasp_gpu+sc142 3 1 1

# [Score-P] src/measurement/profiling/scorep_profile_collapse.c:77: Warning: Score-P callpath depth limitation of 30 exceeded.
# Reached callpath depth was 141

• square scorep-n3N1P/profile.cubex (?)

Filtering (scorep)

• scorep-score -r scorep-n8N1P/profile.cubex

        # Estimated aggregate size of event trace:                   40GB
        # Estimated requirements for largest trace buffer (max_buf): 5GB
        # Estimated memory requirements (SCOREP_TOTAL_MEMORY):       5GB

• sbatch.sh santis 10 vasp_gpu+sc142 3 1 1
• scorep-score -f filterjg profile.cubex

        # Estimated aggregate size of event trace:                   9GB
        # Estimated requirements for largest trace buffer (max_buf): 2827MB
        # Estimated memory requirements (SCOREP_TOTAL_MEMORY):       2897MB

• scorep-score -f filterjg profile.cubex

        # Estimated aggregate size of event trace:                   84MB
        # Estimated requirements for largest trace buffer (max_buf): 28MB
        # Estimated memory requirements (SCOREP_TOTAL_MEMORY):       99MB

• grep EDIFF INCAR

        # EDIFF = 0.1E-04 => DAV: 60
        # EDIFF = 1.      => DAV: 27
        # EDIFF = 500.    => DAV: 06

Tracing (scorep)

• export SCOREP_ENABLE_PROFILING=false
• export SCOREP_ENABLE_TRACING=true
• export SCOREP_CUDA_ENABLE=yes
• export SCOREP_FILTERING_FILE=scorep-n3N1P/filterjg
• export SCOREP_TOTAL_MEMORY=100MB # 16384000
• /apps/ela/vampir/8.5.0/bin/vampir scorep-n3N1T/traces.otf2 (?)