From 9d9e5a51db97f7e5ff4b0d16a35845478cfb6b73 Mon Sep 17 00:00:00 2001 From: Paul Kent Date: Thu, 28 Jan 2016 21:27:53 +0000 Subject: [PATCH] Nightly titan CPU and GPU test jobs. Still under test so they do not yet self-resubmit. Two jobs to keep under 2 hour limit for 1 node jobs git-svn-id: https://subversion.assembla.com/svn/qmcdev/trunk@6719 e5b18d87-469d-4833-9cc0-8cdfa06e9491 --- .../test_automation/nightly_ornl_olcf_eos.job | 6 +- .../nightly_ornl_olcf_titan_cpu.job | 157 ++++++++++++++++++ .../nightly_ornl_olcf_titan_gpu.job | 157 ++++++++++++++++++ 3 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 tests/test_automation/nightly_ornl_olcf_titan_cpu.job create mode 100644 tests/test_automation/nightly_ornl_olcf_titan_gpu.job diff --git a/tests/test_automation/nightly_ornl_olcf_eos.job b/tests/test_automation/nightly_ornl_olcf_eos.job index 31b485643..93bf6b702 100644 --- a/tests/test_automation/nightly_ornl_olcf_eos.job +++ b/tests/test_automation/nightly_ornl_olcf_eos.job @@ -1,7 +1,7 @@ #!/bin/bash # #PBS -A mat044 -#PBS -N Test +#PBS -N QMCPACKTest #PBS -j oe #PBS -l walltime=1:00:00,nodes=1 #PBS -q batch @@ -93,8 +93,10 @@ done if [ -e $place/eos_build_intel/trunk/build_intel/bin/qmcapp ]; then echo "--- Found a built qmcpack executable. Will requeue." if [ -e $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_eos.job ]; then +# Run after 11.01pm next day +qsub -a `date -d "+1 day" +m%d2301` $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_eos.job # Run after 11.01pm each week -qsub -a `date -d "+1 week" +m%d2301` $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_eos.job +#qsub -a `date -d "+1 week" +m%d2301` $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_eos.job # Example for debugging: #qsub -a `date -d "+1 hour" +m%d%H%M` $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_eos.job fi diff --git a/tests/test_automation/nightly_ornl_olcf_titan_cpu.job b/tests/test_automation/nightly_ornl_olcf_titan_cpu.job new file mode 100644 index 000000000..7102aee34 --- /dev/null +++ b/tests/test_automation/nightly_ornl_olcf_titan_cpu.job @@ -0,0 +1,157 @@ +#!/bin/bash +# +#PBS -A mat044 +#PBS -N QMCPACKTestTitan +#PBS -j oe +#PBS -l walltime=2:00:00,nodes=1 +#PBS -q batch +#PBS -m e +#PBS -M kentpr@ornl.gov + +# +# Setup for titan.ccs.ornl.gov +# +# Run the "short" nightlies, requeue if an executable is built. +# +# Titan I/O and compile is incredibly slow -> generous build/test times +# +# Location of job script must be set at end ($HOME/.qmcpack_test_jobs/... ) for resubmit +# +# Checkout, build, and run on scratch. Use custom names for checkout +# directory because scratch is shared. +# + +cd $PBS_O_WORKDIR +module load subversion + +#Must be an absolute path +place=$MEMBERWORK/mat044/QMCPACK_CI_BUILDS_DO_NOT_REMOVE +if [ -e $MEMBERWORK/mat044 ]; then + +if [ ! -e $place ]; then +mkdir $place +fi + +if [ -e $place ]; then + + +for sys in build_gnu build_gnu_complex +do + +cd $place + +if [ -e titan_$sys ]; then +rm -r -f titan_$sys +fi +mkdir titan_$sys +cd titan_$sys + +echo --- Checkout for $sys `date` + +svn checkout https://svn.qmcpack.org/svn/trunk + +# Fake checkout for testing: +#mkdir trunk +#echo >trunk/CMakeLists.txt + +#svn checkout https://subversion.assembla.com/svn/qmcdev/trunk + +if [ -e trunk/CMakeLists.txt ]; then +cd trunk +mkdir $sys +cd $sys +echo --- Building for $sys `date` + +case $sys in +"build_gnu") +# Build with GNU PrgEnv on titan, not default PGI. + + current_prgenv=`echo $LOADEDMODULES|sed -e 's/.*PrgEnv/PrgEnv/' -e 's/\/.*//g'` + echo $current_prgenv + module swap $current_prgenv PrgEnv-gnu + if (echo $LOADEDMODULES | grep -q cudatoolkit) + then + echo "Unloading cudatoolkit for CPU code compilation." + module unload cudatoolkit + fi + + module load cray-hdf5 + module load cmake3 + module load fftw + export FFTW_HOME=$FFTW_DIR/.. + module load boost + export QMCPACK_TEST_SUBMIT_NAME=GNU-Release + ctest -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -S $PWD/../CMake/ctest_script.cmake,release -R short -VV + ;; +"build_gnu_complex") + +# Build with GNU PrgEnv on titan, not default PGI + + current_prgenv=`echo $LOADEDMODULES|sed -e 's/.*PrgEnv/PrgEnv/' -e 's/\/.*//g'` + echo $current_prgenv + module swap $current_prgenv PrgEnv-gnu + if (echo $LOADEDMODULES | grep -q cudatoolkit) + then + echo "Unloading cudatoolkit for CPU code compilation." + module unload cudatoolkit + fi + + module load cray-hdf5 + module load cmake3 + module load fftw + export FFTW_HOME=$FFTW_DIR/.. + module load boost + export QMCPACK_TEST_SUBMIT_NAME=GNU-Complex-Release + ctest -DQMC_COMPLEX=1 -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -S $PWD/../CMake/ctest_script.cmake,release -R short -VV + ;; +"build_gnu_cuda") +# Build with GNU PrgEnv on titan, not default PGI. + current_prgenv=`echo $LOADEDMODULES|sed -e 's/.*PrgEnv/PrgEnv/' -e 's/\/.*//g'` + echo $current_prgenv + module swap $current_prgenv PrgEnv-gnu + + module load cudatoolkit + module load cray-hdf5 + module load cmake3 + module load fftw + export FFTW_HOME=$FFTW_DIR/.. + module load boost + export QMCPACK_TEST_SUBMIT_NAME=GNU-CUDA-Release + ctest -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -DQMC_CUDA=1 -S $PWD/../CMake/ctest_script.cmake,release -R short -VV + ;; +*) + echo "ERROR: Unknown build type $sys" + ;; +esac + +else +echo "ERROR: No CMakeLists. Bad svn checkout." +exit 1 +fi +echo --- Finished $sys `date` +done + +# Hard-coded logic for resubmission +# Default gnu build executable must be present + +if [ -e $place/titan_build_gnu/trunk/build_gnu/bin/qmcapp ]; then +echo "--- Found a built qmcpack executable. Will requeue." +if [ -e $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_titan.job ]; then +echo Would qsub here +# Run after 11.01pm next day +#qsub -a 2301 $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_titan_cpu.job +# Run after one week +#qsub -a `date -d "+1 week" +m%d%H%M` $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_titan_cpu.job +# Example for debugging: +#qsub -a `date -d "+1 hour" +m%d%H%M` $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_titan_cpu.job +fi +else +echo "ERROR: No built qmcpack executable. Will NOT requeue." +fi + +else +echo "ERROR: No directory $place" +exit 1 +fi + +fi diff --git a/tests/test_automation/nightly_ornl_olcf_titan_gpu.job b/tests/test_automation/nightly_ornl_olcf_titan_gpu.job new file mode 100644 index 000000000..d5e45567a --- /dev/null +++ b/tests/test_automation/nightly_ornl_olcf_titan_gpu.job @@ -0,0 +1,157 @@ +#!/bin/bash +# +#PBS -A mat044 +#PBS -N QMCPACKTestTitan +#PBS -j oe +#PBS -l walltime=2:00:00,nodes=1 +#PBS -q batch +#PBS -m e +#PBS -M kentpr@ornl.gov + +# +# Setup for titan.ccs.ornl.gov +# +# Run the "short" nightlies, requeue if an executable is built. +# +# Titan I/O and compile is incredibly slow -> generous build/test times +# +# Location of job script must be set at end ($HOME/.qmcpack_test_jobs/... ) for resubmit +# +# Checkout, build, and run on scratch. Use custom names for checkout +# directory because scratch is shared. +# + +cd $PBS_O_WORKDIR +module load subversion + +#Must be an absolute path +place=$MEMBERWORK/mat044/QMCPACK_CI_BUILDS_DO_NOT_REMOVE +if [ -e $MEMBERWORK/mat044 ]; then + +if [ ! -e $place ]; then +mkdir $place +fi + +if [ -e $place ]; then + + +for sys in build_gnu_cuda +do + +cd $place + +if [ -e titan_$sys ]; then +rm -r -f titan_$sys +fi +mkdir titan_$sys +cd titan_$sys + +echo --- Checkout for $sys `date` + +svn checkout https://svn.qmcpack.org/svn/trunk + +# Fake checkout for testing: +#mkdir trunk +#echo >trunk/CMakeLists.txt + +#svn checkout https://subversion.assembla.com/svn/qmcdev/trunk + +if [ -e trunk/CMakeLists.txt ]; then +cd trunk +mkdir $sys +cd $sys +echo --- Building for $sys `date` + +case $sys in +"build_gnu") +# Build with GNU PrgEnv on titan, not default PGI. + + current_prgenv=`echo $LOADEDMODULES|sed -e 's/.*PrgEnv/PrgEnv/' -e 's/\/.*//g'` + echo $current_prgenv + module swap $current_prgenv PrgEnv-gnu + if (echo $LOADEDMODULES | grep -q cudatoolkit) + then + echo "Unloading cudatoolkit for CPU code compilation." + module unload cudatoolkit + fi + + module load cray-hdf5 + module load cmake3 + module load fftw + export FFTW_HOME=$FFTW_DIR/.. + module load boost + export QMCPACK_TEST_SUBMIT_NAME=GNU-Release + ctest -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -S $PWD/../CMake/ctest_script.cmake,release -R short -VV + ;; +"build_gnu_complex") + +# Build with GNU PrgEnv on titan, not default PGI + + current_prgenv=`echo $LOADEDMODULES|sed -e 's/.*PrgEnv/PrgEnv/' -e 's/\/.*//g'` + echo $current_prgenv + module swap $current_prgenv PrgEnv-gnu + if (echo $LOADEDMODULES | grep -q cudatoolkit) + then + echo "Unloading cudatoolkit for CPU code compilation." + module unload cudatoolkit + fi + + module load cray-hdf5 + module load cmake3 + module load fftw + export FFTW_HOME=$FFTW_DIR/.. + module load boost + export QMCPACK_TEST_SUBMIT_NAME=GNU-Complex-Release + ctest -DQMC_COMPLEX=1 -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -S $PWD/../CMake/ctest_script.cmake,release -R short -VV + ;; +"build_gnu_cuda") +# Build with GNU PrgEnv on titan, not default PGI. + current_prgenv=`echo $LOADEDMODULES|sed -e 's/.*PrgEnv/PrgEnv/' -e 's/\/.*//g'` + echo $current_prgenv + module swap $current_prgenv PrgEnv-gnu + + module load cudatoolkit + module load cray-hdf5 + module load cmake3 + module load fftw + export FFTW_HOME=$FFTW_DIR/.. + module load boost + export QMCPACK_TEST_SUBMIT_NAME=GNU-CUDA-Release + ctest -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -DQMC_CUDA=1 -S $PWD/../CMake/ctest_script.cmake,release -R short -VV + ;; +*) + echo "ERROR: Unknown build type $sys" + ;; +esac + +else +echo "ERROR: No CMakeLists. Bad svn checkout." +exit 1 +fi +echo --- Finished $sys `date` +done + +# Hard-coded logic for resubmission +# Default gnu build executable must be present + +if [ -e $place/titan_build_gnu/trunk/build_gnu/bin/qmcapp ]; then +echo "--- Found a built qmcpack executable. Will requeue." +if [ -e $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_titan.job ]; then +echo Would qsub here +# Run after 11.01pm next day +#qsub -a 2301 $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_titan_gpu.job +# Run after one week +#qsub -a `date -d "+1 week" +m%d%H%M` $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_titan_gpu.job +# Example for debugging: +#qsub -a `date -d "+1 hour" +m%d%H%M` $HOME/.qmcpack_test_jobs/nightly_ornl_olcf_titan_gpu.job +fi +else +echo "ERROR: No built qmcpack executable. Will NOT requeue." +fi + +else +echo "ERROR: No directory $place" +exit 1 +fi + +fi