CP2K (perftools)

[jg piccinali]

Santis + Cuda/6.5

Setup

module swap PrgEnv-cray PrgEnv-gnu
module load craype-accel-nvidia35+cuda65
module load fftw
export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH

module  use /project/csstaff/proposals
module load perflite/622cuda
module load craype-hugepages8M

Compile

cd /apps/daint/sandbox/jgp/cp2k/svn/cp2k.notool+cuda65/
make ARCH=CRAY-XC30-gfortran-cuda VERSION=psmp

Run

cd /apps/daint/sandbox/jgp/cp2k/svn/cp2k.notool+cuda65/
cd exe/CRAY-XC30-gfortran-cuda/08CN
./sbatch.sh santis 10 ../cp2k.psmp 8 1 8 H2O-dft-ls_NREP2.inp 

export CRAY_CUDA_MPS=1  # without perftool
export CRAY_CUDA_MPS=0  # with perftool

Not enough memory for NREP5 on Santis

[jg piccinali]

mpi+cuda test

[jg piccinali]

CP2K + PERFLITE/622

• export CRAY_CUDA_MPS=0
• aprun -n 8 -N 1 -d 8 -j 1 ../cp2k.psmp H2O-dft-ls_NREP2.inp

but issue with craype-hugepages8M

libhugetlbfs [nid00021:15894]: WARNING: Unable to verify address range 0x42e43000 - 0x43000000.  Not empty?
libhugetlbfscp2k.psmp: malloc.c:3091: sYSMALLOc: 
Assertion `(old_top == (((mbinptr) (((char *) &((av)->bins[((1) - 1) * 2])) - 
__builtin_offsetof (struct malloc_chunk, fd)))) && old_size == 0) || 
((unsigned long) (old_size) >= (unsigned long)((((__builtin_offsetof (struct malloc_chunk, fd_nextsize))
+((2 * (sizeof(size_t))) - 1)) & ~((2 * (sizeof(size_t))) - 1))) && 
((old_top)->size & 0x1) && ((unsigned long)old_end & pagemask) == 0)' failed. 

[jg piccinali]

• Issue with craype-hugepages8M still present
• Without craype-hugepages8M, ok:

[jg piccinali]

Santis + Cuda/6.5 + MPS=1

perftools-lite/6.2.3 supports CRAY_CUDA_MPS=1

[jg piccinali]

Daint

module swap PrgEnv-cray PrgEnv-gnu
module load craype-accel-nvidia35
module load fftw
module load craype-hugepages8M
module  use /project/csstaff/proposals
module load perflite/625cuda

cd /apps/daint/5.2.UP02/sandbox/jgp/cp2k/GNU482/cp2k-code-15721-trunk/cp2k/makefiles/
vim CRAY-XC30-gfortran-gpu.psmp
make ARCH=CRAY-XC30-gfortran-gpu VERSION=psmp   clean
make ARCH=CRAY-XC30-gfortran-gpu VERSION=psmp   -j8

cd ../exe/CRAY-XC30-gfortran-gpu 
export CRAY_CUDA_MPS=1 
sbatch.sh daint 10 ./cp2k.psmp 8 1 8 H2O-dft-ls_NREP2.inp "" "" -Ausup 

Currently Loaded Modulefiles:
modules/3.2.10.3
nodestat/2.2-1.0502.53712.3.109.ari
sdb/1.0-1.0502.55976.5.27.ari
alps/5.2.1-2.0502.9041.11.6.ari
lustre-cray_ari_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19897.18.2
udreg/2.3.2-1.0502.9275.1.12.ari
ugni/5.0-1.0502.9685.4.24.ari
gni-headers/3.0-1.0502.9684.5.2.ari
dmapp/7.0.1-1.0502.11083.8.4.ari
xpmem/0.1-2.0502.55507.3.2.ari
hss-llm/7.2.0
Base-opts/1.0.2-1.0502.53325.1.2.ari
craype-sandybridge
craype-network-aries
craype/2.4.0
slurm
cray-mpich/7.2.2
ddt/5.0
gcc/4.8.2
totalview-support/1.1.4
totalview/8.11.0
cray-libsci/13.0.4
pmi/5.0.7-1.0000.10678.155.25.ari
atp/1.8.2
PrgEnv-gnu/5.2.40
cray-libsci_acc/3.1.1
cudatoolkit/6.5.14-1.0502.9613.6.1
craype-accel-nvidia35
rca/1.0.0-2.0502.53711.3.127.ari
perflite/625cuda
fftw/3.3.4.3

[jg piccinali]

• sbatch.sh santis 10 ./cp2k.psmp 64 8 1 H2O-dft-ls_NREP2.inp