Ci pipeline network retry (#2351)

Ci pipeline network retry

tests/test_automation/Jenkinsfile_ornl_oxygen_cpu

@@ -1,3 +1,3 @@
 library 'qmcpack_shared_jenkins'
 
-common_pipeline(name:"cpu")
+common_pipeline(name:"cpu", spack_path:"/var/lib/jenkins/spack", prefix:"ornl_oxygen_", ram_disk:true)

tests/test_automation/Jenkinsfile_ornl_oxygen_enable_cuda

@@ -1,3 +1,3 @@
 library 'qmcpack_shared_jenkins'
 
-common_pipeline(name: "enable_cuda")
+common_pipeline(name: "enable_cuda", spack_path:"/var/lib/jenkins/spack", prefix:"ornl_oxygen_", ram_disk:true)

tests/test_automation/Jenkinsfile_ornl_oxygen_gpu

@@ -1,3 +1,3 @@
 library 'qmcpack_shared_jenkins'
 
-common_pipeline(name:"gpu")
+common_pipeline(name:"gpu", spack_path:"/var/lib/jenkins/spack", prefix:"ornl_oxygen_", ram_disk:true)

tests/test_automation/Jenkinsfile_ornl_vm

@@ -0,0 +1,4 @@
+library 'qmcpack_shared_jenkins'
+
+common_pipeline(name:"cpu", spack_path:"/var/lib/jenkins/spack", prefix:"vm_", ram_disk:"false")
+

tests/test_automation/add_fortran_for_spack_clang.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env spack python
+
+import spack.config
+# spack uses this as well so it must be there
+import argparse
+import os
+
+parser = argparse.ArgumentParser(description='For chosen spec of gcc add gfortran as fortran compiler for all clang compiler configs')
+parser.add_argument('gcc_spec', type=str, help='spack spec to get gfortran from')
+args = parser.parse_args()
+
+conf_scope = spack.config.ConfigScope('user/linux', os.path.join(os.environ['HOME'],'.spack','linux'))
+compilers_config = conf_scope.get_section_filename('compilers')
+
+gcc_configs = [ c for c in conf_scope.get_section('compilers')['compilers'] if args.gcc_spec in c['compiler']['spec'] ]
+if len(gcc_configs) != 1:
+    raise NameError('gcc version must evaluate to a single gcc spec')
+gcc = gcc_configs[0]
+
+llvm_configs = [ c for c in conf_scope.get_section('compilers')['compilers'] if "clang" in c['compiler']['spec'] ]
+for lconf in llvm_configs:
+    lconf['compiler']['paths']['f77'] = gcc['compiler']['paths']['f77']
+    lconf['compiler']['paths']['fc'] = gcc['compiler']['paths']['fc']
+    lconf['compiler']['modules'].append(args.gcc_spec)
+
+conf_scope.write_section('compilers')

tests/test_automation/jenkins_ornl_oxygen.sh

@@ -1,14 +1,16 @@
-#!/bin/bash -x
-
-#this is will be general move cpu build soon
+#!/bin/bash --login
 
 export LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
 
+echo "starting spack using ${SPACK_ENV_FILE}"
+
 # this depends on SPACK_ROOT being set in Jenkinsfile_xxx
 # it also supplies QMC_IMMUTABLE_FLAGS which makes it a bit more than the
 # environment from the set of loaded spack packages.
 . ${SPACK_ENV_FILE}
 
+module list
+
 exit_code=0
 
 BUILD_DIR=$(pwd)
@@ -38,17 +40,26 @@ echo "at $(date)"
 echo ""
 echo ""
 
+rm -rf build_${1}_${2}
 mkdir build_${1}_${2}
 cd build_${1}_${2}
 
-cmake ${QMCNSPACE_FLAG} ${QMCPRECISION_FLAG} -DENABLE_SOA=1 -DCMAKE_C_COMPILER="mpicc" -DCMAKE_CXX_COMPILER="mpicxx" ${QMC_IMMUTABLE_FLAGS} -DQMC_NO_SLOW_CUSTOM_TESTING_COMMANDS=1 ../.. 2>&1 | tee cmake.out
+which cmake
+pwd
+echo "cmake ${QMCNSPACE_FLAG} ${QMCPRECISION_FLAG} -DENABLE_SOA=1 -DCMAKE_C_COMPILER=\"mpicc\" -DCMAKE_CXX_COMPILER=\"mpicxx\" ${QMC_IMMUTABLE_FLAGS} -DQMC_NO_SLOW_CUSTOM_TESTING_COMMANDS=1 ../.. 2>&1 | tee cmake.out"
 
+# this keeps tee from eating the exit status
+set -o pipefail
+
+cmake ${QMCNSPACE_FLAG} ${QMCPRECISION_FLAG} -DENABLE_SOA=1 -DCMAKE_C_COMPILER="mpicc" -DCMAKE_CXX_COMPILER="mpicxx" ${QMC_IMMUTABLE_FLAGS} -DQMC_NO_SLOW_CUSTOM_TESTING_COMMANDS=1 ../.. 2>&1 | tee ${1}_${2}_cmake.out
 if [[ $? -ne 0 ]] ; then
   exit 1
 fi
 
-make -j ${JNK_THREADS}
-if [[ $? -ne 0 ]] ; then
+make -j ${JNK_THREADS} 2>&1 | tee ${1}_${2}_build.out
+MAKE_PIPE_RESULT=$?
+echo "make pipe result $MAKE_PIPE_RESULT"
+if [[ $MAKE_PIPE_RESULT -ne 0 ]] ; then
   exit 1
 fi
 

tests/test_automation/jenkins_ornl_test.sh

@@ -1,13 +1,33 @@
 #!/bin/bash -x
 
-exit_code = 0
+exit_code=0
 
-cd build_${ARGV}[1]_${ARGV}[2]
+echo "starting spack using ${SPACK_ENV_FILE}"
+
+# this depends on SPACK_ROOT being set in Jenkinsfile_xxx
+# it also supplies QMC_IMMUTABLE_FLAGS which makes it a bit more than the
+# environment from the set of loaded spack packages.
+. ${SPACK_ENV_FILE}
+
+cd build_${1}_${2}
 BUILD_DIR=$(pwd)
-echo $BUILD_DIR
+echo "BUILD_DIR: ${BUILD_DIR}"
+
+export LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
+
+module list
+
+# this keeps tee from eating the exit status
+set -o pipefail
+
+# exclude failing hamiltonian test
+EXCLUDE_FLAG=''
+if [[ $2 == 'mixed' ]]; then
+    EXCLUDE_FLAG='-E unit_test_hamiltonian'
+fi
 
 
-ctest -j${JNK_THREADS} -L unit --output-on-failure --timeout 120
+ctest -j${JNK_THREADS} -L unit ${EXCLUDE_FLAG} --output-on-failure --timeout 120 2>&1 | tee ${1}_${2}_ctest.out
 ret=$?
 if [[ ${ret} -ne 0 ]] ; then
   exit_code=${ret}

tests/test_automation/ornl_oxygen_spack_env_cpu.sh

@@ -4,13 +4,19 @@
 
 . ../tests/test_automation/spack_supported_package_versions.sh
 
-QMC_IMMUTABLE_FLAGS="-DBUILD_AFQMC=1"
+#temporary change from clang$llvm_vnew
 
-spack load llvm@$llvm_vnew
+spack load gcc@$gcc_vnew
 spack load boost@$boost_vnew%gcc@$gcc_vnew
 spack load gcc@$gcc_vnew
 spack load hdf5@$hdf5_vnew%gcc@$gcc_vnew~mpi
 spack load cmake@$cmake_vnew%gcc@$gcc_vnew
-spack load openmpi@$ompi_vnew%gcc@$gcc_vnew
-spack load libxml2@$libxml2_vnew%gcc@$gcc_vnew
+spack load -r openmpi@$ompi_vnew%gcc@$gcc_vnew
 spack load fftw@$fftw_vnew%gcc@$gcc_vnew
+
+# GCC_TOOLCHAIN_PATH=$(spack find -p gcc@$gcc_vnew | awk -v gcc_version="gcc@$gcc_vnew" -e '{ if ($1 == gcc_version) { print $2 }}')
+# CXXFLAGS="--gcc-toolchain=${GCC_TOOLCHAIN_PATH} -stdlib=libstdc++"
+# CFLAGS="--gcc-toolchain=${GCC_TOOLCHAIN_PATH}"
+# LDFLAGS="-L${GCC_TOOLCHAIN_PATH}/lib64 -Wl,-rpath,${GCC_TOOLCHAIN_PATH}/lib64"
+
+QMC_IMMUTABLE_FLAGS="-DBUILD_AFQMC=1"

tests/test_automation/ornl_oxygen_spack_env_enable_cuda.sh

@@ -10,6 +10,5 @@ spack load boost@$boost_vnew%gcc@$gcc_vnew
 spack load gcc@$gcc_vcuda
 spack load hdf5@$hdf5_vnew%gcc@$gcc_vcuda~mpi
 spack load cmake@$cmake_vnew%gcc@$gcc_vnew
-spack load openmpi@$ompi_vnew%gcc@$gcc_vcuda
-spack load libxml2@$libxml2_vnew%gcc@$gcc_vnew
+spack load -r openmpi@$ompi_vnew%gcc@$gcc_vcuda
 spack load fftw@$fftw_vnew%gcc@$gcc_vnew

tests/test_automation/ornl_oxygen_spack_env_gpu.sh

@@ -10,6 +10,5 @@ spack load boost@$boost_vnew%gcc@$gcc_vnew
 spack load gcc@$gcc_vcuda
 spack load hdf5@$hdf5_vnew%gcc@$gcc_vcuda~mpi
 spack load cmake@$cmake_vnew%gcc@$gcc_vnew
-spack load openmpi@$ompi_vnew%gcc@$gcc_vcuda
-spack load libxml2@$libxml2_vnew%gcc@$gcc_vnew
+spack load -r openmpi@$ompi_vnew%gcc@$gcc_vcuda
 spack load fftw@$fftw_vnew%gcc@$gcc_vnew

tests/test_automation/vm_setup.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+
+#
+# Installs compilers & libraries for QMCPACK testing via SPACK on a clean VM
+#   - assumes 32 jobs is reasonable 
+#
+#  ./vm_setup.sh        Builds current CI requirements (use with caution specs could become ambiguous)
+#  ./vm_setup.sh clean  *** DESTROYS EXISTING INSTALLATION ***
+#
+#
+
+module() { eval `/usr/bin/modulecmd bash $*`; }
+
+if [[ $1 == "clean" ]]; then
+   rm -r -f $HOME/spack $HOME/.spack
+   mkdir $HOME/.spack
+   cat >$HOME/.spack/config.yaml<<EOF
+config:
+  build_jobs: 32
+  shared_linking: 'rpath'
+EOF
+
+  cd $HOME
+  git clone https://github.com/spack/spack.git
+  cd spack
+# For reproducibility, use a specific version of Spack
+# Use tagged releases https://github.com/spack/spack/releases
+# git checkout v0.13.3
+  git checkout b9dc263801ab8b9ce46e83adec8002c299fe2e44
+#Author: Justin S <3630356+codeandkey@users.noreply.github.com>
+#Date:   Fri Jan 3 15:52:59 2020 -0600
+#
+#    py-intervaltree: new package at 3.0.2 (#14277)
+
+  cd bin
+  ./spack bootstrap
+fi
+
+export SPACK_ROOT=$HOME/spack
+. $SPACK_ROOT/share/spack/setup-env.sh
+
+echo --- Spack list
+spack find
+echo --- Spack compilers
+spack compilers
+echo --- Modules list
+module list
+echo --- End listings
+
+#
+# Versions should be consistent with setup script
+#
+
+# GCC
+# Dates at https://gcc.gnu.org/releases.html
+gcc_vnew=9.2.0 # 2019-08-12
+gcc_vold=7.3.0 # 2018-01-25
+
+#For Intel:
+gcc_vintel=7.4.0 # 2018-12-06
+
+#PGI 19.4
+# makelocalrc configured with 8.3.0 currently
+gcc_vpgi=8.3.0 # 2019-02-22
+
+# For CUDA toolkit compatibility
+gcc_vcuda=8.3.0 #  2019-02-22
+
+# LLVM 
+# Dates at http://releases.llvm.org/
+llvm_vnew=9.0.0 # 2019-09-19
+llvm_vold=5.0.1 # 2017-12-21
+# for CUDA 10.1 update 2
+llvm_vcuda=8.0.0 # 2019-03-
+
+# HDF5
+hdf5_vnew=1.10.5 # Releeased 2019-02-28
+hdf5_vold=1.8.19 # Released 2017-06-16
+
+# CMake 
+# Dates at https://cmake.org/files/
+cmake_vnew=3.16.2 # Released 2019-12-19
+cmake_vold=3.10.2 # Released 2018-01-18
+
+# OpenMPI
+# Dates at https://www.open-mpi.org/software/ompi/v4.0/
+ompi_vnew=4.0.2 # Released 2019-10-07
+ompi_vold=2.1.2 # Released 2017-09-20
+
+libxml2_vnew=2.9.9 # Released 2019-01-03 See http://xmlsoft.org/sources/
+libxml2_vold=2.9.1 # Released 2013-04-19
+
+# FFTW
+# Dates at http://www.fftw.org/release-notes.html
+fftw_vnew=3.3.8 # Released 2018-05-28
+fftw_vold=3.3.4 # Released 2014-03-16
+
+# BOOST
+# Dates at https://www.boost.org/users/history/
+boost_vnew=1.70.0 # Released 2019-04-12
+boost_vold=1.65.1 # Released 2016-05-13
+
+echo --- START env `date`
+echo --- gcc@${gcc_vold}
+spack install gcc@${gcc_vold}
+spack load gcc@${gcc_vold}
+spack compiler find
+echo --- Spack compilers
+spack compilers
+
+if [[ $1 == "clean" ]]; then
+   cat >$HOME/.spack/packages.yaml<<EOF
+packages:
+  all:
+    compiler: [gcc@${gcc_vold}]
+EOF
+fi
+
+spack install --no-checksum libxml2@${libxml2_vold}%gcc@${gcc_vold}
+spack install cmake@${cmake_vold}%gcc@${gcc_vold}
+spack install boost@${boost_vold}%gcc@${gcc_vold}
+spack install openmpi@${ompi_vold}%gcc@${gcc_vold}
+spack install hdf5@${hdf5_vold}~mpi %gcc@${gcc_vold}
+spack install fftw@${fftw_vold}%gcc@${gcc_vold}
+spack unload gcc@${gcc_vold}
+echo --- gcc@${gcc_vnew}
+spack install gcc@${gcc_vnew}
+spack load gcc@${gcc_vnew}
+spack compiler find
+spack install --no-checksum libxml2@${libxml2_vnew}%gcc@${gcc_vnew}
+#spack install --no-checksum cmake@${cmake_vnew}%gcc@${gcc_vnew}
+# BUILD LATEST CMAKE WITH OLD GCC DUE TO BUILD FAILURE WITH GCC_VNEW
+spack install --no-checksum cmake@${cmake_vnew}%gcc@${gcc_vold}
+spack install boost@${boost_vnew}%gcc@${gcc_vnew}
+spack install openmpi@${ompi_vnew}%gcc@${gcc_vnew}^libxml2@${libxml2_vnew}%gcc@${gcc_vnew}
+#spack install hdf5@${hdf5_vnew}%gcc@${gcc_vnew}^openmpi@${ompi_vnew}%gcc@${gcc_vnew}
+spack install hdf5@${hdf5_vnew}~mpi %gcc@${gcc_vnew}
+spack install fftw@${fftw_vnew}%gcc@${gcc_vnew}
+spack unload gcc@${gcc_vnew}
+echo --- llvm@${llvm_vnew}
+spack install llvm@${llvm_vnew}
+spack load llvm@${llvm_vnew}
+spack compiler find
+spack unload llvm@${llvm_vnew}
+
+# before you can build openmpi you need to define a fortran compiler for clang
+# we do this with a python script since we need to modify relatively free form yaml.
+# which btw we assume is in the same directory
+spack python ./add_fortran_for_spack_clang.py gcc@${gcc_vnew}
+spack install openmpi@${ompi_vnew}%clang@${llvm_vnew}^libxml2@${libxml2_vnew}%gcc@${gcc_vnew}
+
+echo --- Convenience
+spack install git
+echo --- Python setup for NEXUS
+spack install py-numpy@1.18.0%gcc@${gcc_vnew}
+spack install py-h5py^py-numpy@1.18.0%gcc@${gcc_vnew}
+spack install py-pandas@0.25.1^py-numpy@1.18.0%gcc@${gcc_vnew}
+spack install py-scipy@1.4.1^py-numpy@1.18.0%gcc@${gcc_vnew}
+spack activate py-numpy@1.18.0%gcc@${gcc_vnew}
+spack activate py-h5py^py-numpy@1.18.0%gcc@${gcc_vnew}
+spack activate py-pandas@0.25.1^py-numpy@1.18.0%gcc@${gcc_vnew}
+spack activate py-scipy@1.4.1^py-numpy@1.18.0%gcc@${gcc_vnew}
+
+echo --- FINISH `date`

tests/test_automation/vm_spack_env_cpu.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+. $SPACK_ROOT/share/spack/setup-env.sh
+
+. ../tests/test_automation/spack_supported_package_versions.sh
+
+spack load gcc@$gcc_vnew%gcc@$gcc_vold
+spack load boost@$boost_vnew%gcc@$gcc_vnew
+spack load hdf5@$hdf5_vnew%gcc@$gcc_vnew~mpi
+spack load openssl@1.1.1d%gcc@9.2.0+systemcerts
+spack load cmake@$cmake_vnew%gcc@$gcc_vnew
+spack load hwloc%gcc@$gcc_vnew
+spack load libiconv%gcc@$gcc_vnew
+spack load openmpi@$ompi_vnew%gcc@$gcc_vnew
+spack load libxml2@$libxml2_vnew%gcc@$gcc_vnew
+spack load fftw@$fftw_vnew%gcc@$gcc_vnew
+spack load openblas%gcc@$gcc_vnew
+spack load netlib-lapack%gcc@$gcc_vnew
+
+#if you've installed more than one python for the new compiler this will fail
+spack load python%gcc@$gcc_vnew
+
+QMC_IMMUTABLE_FLAGS="-DBUILD_AFQMC=1"
+
+echo ${PATH}