GPU direct

[jg piccinali]

Kescha

Setup

• module load gcc/4.8.2 mvapich2gdr_gnu/2.1rc2
• module load craype-accel-nvidia35

# Currently Loaded Modulefiles:
# craype-haswell
# cudatoolkit/6.5.14
# gcc/4.8.2
# mvapich2gdr_gnu/2.1rc2
# craype-accel-nvidia35

Compile

• cd /apps/escha/mchquickstart.git/mpicuda/GDR/osu-micro-benchmarks-5.0/

./configure \
--enable-cuda \
--prefix=gnu482-mvapich2gdr2.1rc2 \
LDFLAGS=-L/opt/local/slurm/default/lib64

• mpicc -c -I. -O2 osu_pt2pt.c
• mpicc -I. -O2 osu_pt2pt.o osu_bw.c -o GNU+not -L/opt/local/slurm/default/lib64

Run (nvprof)

• srun --gres=gpu:2 --exclusive -n2 nvprof -o nvprof.output.%h.%p ./GNU+not D D
• srun --gres=gpu:2 --exclusive -n2 ./GNU H D

Run (scorep/1.4.2)

• scorep --mpp=mpi --cuda nvcc -arch=sm_37 -c ../src/mpicu.cu
• scorep --mpp=mpi --cuda mpicc -c ../src/mpic.c
• scorep --mpp=mpi --cuda mpicc mpic.o mpicu.o -L/opt/local/slurm/default/lib64
• export SCOREP_ENABLE_PROFILING=false
• export SCOREP_ENABLE_TRACING=true
• export SCOREP_CUDA_ENABLE=yes
• srun --gres=gpu:2 --exclusive -n2 ./GNU+sc142 H D
• srun --gres=gpu:2 --exclusive -n2 ./GNU+sc142 D D