mirror of https://github.com/QMCPACK/qmcpack.git
parent
c49b2a5857
commit
d54c07d72c
|
@ -1,3 +1,3 @@
|
|||
library 'qmcpack_shared_jenkins'
|
||||
|
||||
common_pipeline(name:"cpu")
|
||||
common_pipeline(name:"cpu", spack_path:"/var/lib/jenkins/spack", prefix:"ornl_oxygen_", ram_disk:true)
|
||||
|
|
|
@ -1,3 +1,3 @@
|
|||
library 'qmcpack_shared_jenkins'
|
||||
|
||||
common_pipeline(name: "enable_cuda")
|
||||
common_pipeline(name: "enable_cuda", spack_path:"/var/lib/jenkins/spack", prefix:"ornl_oxygen_", ram_disk:true)
|
||||
|
|
|
@ -1,3 +1,3 @@
|
|||
library 'qmcpack_shared_jenkins'
|
||||
|
||||
common_pipeline(name:"gpu")
|
||||
common_pipeline(name:"gpu", spack_path:"/var/lib/jenkins/spack", prefix:"ornl_oxygen_", ram_disk:true)
|
||||
|
|
|
@ -0,0 +1,4 @@
|
|||
library 'qmcpack_shared_jenkins'
|
||||
|
||||
common_pipeline(name:"cpu", spack_path:"/var/lib/jenkins/spack", prefix:"vm_", ram_disk:"false")
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
#!/usr/bin/env spack python
|
||||
|
||||
import spack.config
|
||||
# spack uses this as well so it must be there
|
||||
import argparse
|
||||
import os
|
||||
|
||||
parser = argparse.ArgumentParser(description='For chosen spec of gcc add gfortran as fortran compiler for all clang compiler configs')
|
||||
parser.add_argument('gcc_spec', type=str, help='spack spec to get gfortran from')
|
||||
args = parser.parse_args()
|
||||
|
||||
conf_scope = spack.config.ConfigScope('user/linux', os.path.join(os.environ['HOME'],'.spack','linux'))
|
||||
compilers_config = conf_scope.get_section_filename('compilers')
|
||||
|
||||
gcc_configs = [ c for c in conf_scope.get_section('compilers')['compilers'] if args.gcc_spec in c['compiler']['spec'] ]
|
||||
if len(gcc_configs) != 1:
|
||||
raise NameError('gcc version must evaluate to a single gcc spec')
|
||||
gcc = gcc_configs[0]
|
||||
|
||||
llvm_configs = [ c for c in conf_scope.get_section('compilers')['compilers'] if "clang" in c['compiler']['spec'] ]
|
||||
for lconf in llvm_configs:
|
||||
lconf['compiler']['paths']['f77'] = gcc['compiler']['paths']['f77']
|
||||
lconf['compiler']['paths']['fc'] = gcc['compiler']['paths']['fc']
|
||||
lconf['compiler']['modules'].append(args.gcc_spec)
|
||||
|
||||
conf_scope.write_section('compilers')
|
|
@ -1,14 +1,16 @@
|
|||
#!/bin/bash -x
|
||||
|
||||
#this is will be general move cpu build soon
|
||||
#!/bin/bash --login
|
||||
|
||||
export LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
|
||||
|
||||
echo "starting spack using ${SPACK_ENV_FILE}"
|
||||
|
||||
# this depends on SPACK_ROOT being set in Jenkinsfile_xxx
|
||||
# it also supplies QMC_IMMUTABLE_FLAGS which makes it a bit more than the
|
||||
# environment from the set of loaded spack packages.
|
||||
. ${SPACK_ENV_FILE}
|
||||
|
||||
module list
|
||||
|
||||
exit_code=0
|
||||
|
||||
BUILD_DIR=$(pwd)
|
||||
|
@ -38,17 +40,26 @@ echo "at $(date)"
|
|||
echo ""
|
||||
echo ""
|
||||
|
||||
rm -rf build_${1}_${2}
|
||||
mkdir build_${1}_${2}
|
||||
cd build_${1}_${2}
|
||||
|
||||
cmake ${QMCNSPACE_FLAG} ${QMCPRECISION_FLAG} -DENABLE_SOA=1 -DCMAKE_C_COMPILER="mpicc" -DCMAKE_CXX_COMPILER="mpicxx" ${QMC_IMMUTABLE_FLAGS} -DQMC_NO_SLOW_CUSTOM_TESTING_COMMANDS=1 ../.. 2>&1 | tee cmake.out
|
||||
which cmake
|
||||
pwd
|
||||
echo "cmake ${QMCNSPACE_FLAG} ${QMCPRECISION_FLAG} -DENABLE_SOA=1 -DCMAKE_C_COMPILER=\"mpicc\" -DCMAKE_CXX_COMPILER=\"mpicxx\" ${QMC_IMMUTABLE_FLAGS} -DQMC_NO_SLOW_CUSTOM_TESTING_COMMANDS=1 ../.. 2>&1 | tee cmake.out"
|
||||
|
||||
# this keeps tee from eating the exit status
|
||||
set -o pipefail
|
||||
|
||||
cmake ${QMCNSPACE_FLAG} ${QMCPRECISION_FLAG} -DENABLE_SOA=1 -DCMAKE_C_COMPILER="mpicc" -DCMAKE_CXX_COMPILER="mpicxx" ${QMC_IMMUTABLE_FLAGS} -DQMC_NO_SLOW_CUSTOM_TESTING_COMMANDS=1 ../.. 2>&1 | tee ${1}_${2}_cmake.out
|
||||
if [[ $? -ne 0 ]] ; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
make -j ${JNK_THREADS}
|
||||
if [[ $? -ne 0 ]] ; then
|
||||
make -j ${JNK_THREADS} 2>&1 | tee ${1}_${2}_build.out
|
||||
MAKE_PIPE_RESULT=$?
|
||||
echo "make pipe result $MAKE_PIPE_RESULT"
|
||||
if [[ $MAKE_PIPE_RESULT -ne 0 ]] ; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
@ -1,13 +1,33 @@
|
|||
#!/bin/bash -x
|
||||
|
||||
exit_code = 0
|
||||
exit_code=0
|
||||
|
||||
cd build_${ARGV}[1]_${ARGV}[2]
|
||||
echo "starting spack using ${SPACK_ENV_FILE}"
|
||||
|
||||
# this depends on SPACK_ROOT being set in Jenkinsfile_xxx
|
||||
# it also supplies QMC_IMMUTABLE_FLAGS which makes it a bit more than the
|
||||
# environment from the set of loaded spack packages.
|
||||
. ${SPACK_ENV_FILE}
|
||||
|
||||
cd build_${1}_${2}
|
||||
BUILD_DIR=$(pwd)
|
||||
echo $BUILD_DIR
|
||||
echo "BUILD_DIR: ${BUILD_DIR}"
|
||||
|
||||
export LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
|
||||
|
||||
module list
|
||||
|
||||
# this keeps tee from eating the exit status
|
||||
set -o pipefail
|
||||
|
||||
# exclude failing hamiltonian test
|
||||
EXCLUDE_FLAG=''
|
||||
if [[ $2 == 'mixed' ]]; then
|
||||
EXCLUDE_FLAG='-E unit_test_hamiltonian'
|
||||
fi
|
||||
|
||||
|
||||
ctest -j${JNK_THREADS} -L unit --output-on-failure --timeout 120
|
||||
ctest -j${JNK_THREADS} -L unit ${EXCLUDE_FLAG} --output-on-failure --timeout 120 2>&1 | tee ${1}_${2}_ctest.out
|
||||
ret=$?
|
||||
if [[ ${ret} -ne 0 ]] ; then
|
||||
exit_code=${ret}
|
||||
|
|
|
@ -4,13 +4,19 @@
|
|||
|
||||
. ../tests/test_automation/spack_supported_package_versions.sh
|
||||
|
||||
QMC_IMMUTABLE_FLAGS="-DBUILD_AFQMC=1"
|
||||
#temporary change from clang$llvm_vnew
|
||||
|
||||
spack load llvm@$llvm_vnew
|
||||
spack load gcc@$gcc_vnew
|
||||
spack load boost@$boost_vnew%gcc@$gcc_vnew
|
||||
spack load gcc@$gcc_vnew
|
||||
spack load hdf5@$hdf5_vnew%gcc@$gcc_vnew~mpi
|
||||
spack load cmake@$cmake_vnew%gcc@$gcc_vnew
|
||||
spack load openmpi@$ompi_vnew%gcc@$gcc_vnew
|
||||
spack load libxml2@$libxml2_vnew%gcc@$gcc_vnew
|
||||
spack load -r openmpi@$ompi_vnew%gcc@$gcc_vnew
|
||||
spack load fftw@$fftw_vnew%gcc@$gcc_vnew
|
||||
|
||||
# GCC_TOOLCHAIN_PATH=$(spack find -p gcc@$gcc_vnew | awk -v gcc_version="gcc@$gcc_vnew" -e '{ if ($1 == gcc_version) { print $2 }}')
|
||||
# CXXFLAGS="--gcc-toolchain=${GCC_TOOLCHAIN_PATH} -stdlib=libstdc++"
|
||||
# CFLAGS="--gcc-toolchain=${GCC_TOOLCHAIN_PATH}"
|
||||
# LDFLAGS="-L${GCC_TOOLCHAIN_PATH}/lib64 -Wl,-rpath,${GCC_TOOLCHAIN_PATH}/lib64"
|
||||
|
||||
QMC_IMMUTABLE_FLAGS="-DBUILD_AFQMC=1"
|
||||
|
|
|
@ -10,6 +10,5 @@ spack load boost@$boost_vnew%gcc@$gcc_vnew
|
|||
spack load gcc@$gcc_vcuda
|
||||
spack load hdf5@$hdf5_vnew%gcc@$gcc_vcuda~mpi
|
||||
spack load cmake@$cmake_vnew%gcc@$gcc_vnew
|
||||
spack load openmpi@$ompi_vnew%gcc@$gcc_vcuda
|
||||
spack load libxml2@$libxml2_vnew%gcc@$gcc_vnew
|
||||
spack load -r openmpi@$ompi_vnew%gcc@$gcc_vcuda
|
||||
spack load fftw@$fftw_vnew%gcc@$gcc_vnew
|
||||
|
|
|
@ -10,6 +10,5 @@ spack load boost@$boost_vnew%gcc@$gcc_vnew
|
|||
spack load gcc@$gcc_vcuda
|
||||
spack load hdf5@$hdf5_vnew%gcc@$gcc_vcuda~mpi
|
||||
spack load cmake@$cmake_vnew%gcc@$gcc_vnew
|
||||
spack load openmpi@$ompi_vnew%gcc@$gcc_vcuda
|
||||
spack load libxml2@$libxml2_vnew%gcc@$gcc_vnew
|
||||
spack load -r openmpi@$ompi_vnew%gcc@$gcc_vcuda
|
||||
spack load fftw@$fftw_vnew%gcc@$gcc_vnew
|
||||
|
|
|
@ -0,0 +1,164 @@
|
|||
#!/bin/bash
|
||||
|
||||
#
|
||||
# Installs compilers & libraries for QMCPACK testing via SPACK on a clean VM
|
||||
# - assumes 32 jobs is reasonable
|
||||
#
|
||||
# ./vm_setup.sh Builds current CI requirements (use with caution specs could become ambiguous)
|
||||
# ./vm_setup.sh clean *** DESTROYS EXISTING INSTALLATION ***
|
||||
#
|
||||
#
|
||||
|
||||
module() { eval `/usr/bin/modulecmd bash $*`; }
|
||||
|
||||
if [[ $1 == "clean" ]]; then
|
||||
rm -r -f $HOME/spack $HOME/.spack
|
||||
mkdir $HOME/.spack
|
||||
cat >$HOME/.spack/config.yaml<<EOF
|
||||
config:
|
||||
build_jobs: 32
|
||||
shared_linking: 'rpath'
|
||||
EOF
|
||||
|
||||
cd $HOME
|
||||
git clone https://github.com/spack/spack.git
|
||||
cd spack
|
||||
# For reproducibility, use a specific version of Spack
|
||||
# Use tagged releases https://github.com/spack/spack/releases
|
||||
# git checkout v0.13.3
|
||||
git checkout b9dc263801ab8b9ce46e83adec8002c299fe2e44
|
||||
#Author: Justin S <3630356+codeandkey@users.noreply.github.com>
|
||||
#Date: Fri Jan 3 15:52:59 2020 -0600
|
||||
#
|
||||
# py-intervaltree: new package at 3.0.2 (#14277)
|
||||
|
||||
cd bin
|
||||
./spack bootstrap
|
||||
fi
|
||||
|
||||
export SPACK_ROOT=$HOME/spack
|
||||
. $SPACK_ROOT/share/spack/setup-env.sh
|
||||
|
||||
echo --- Spack list
|
||||
spack find
|
||||
echo --- Spack compilers
|
||||
spack compilers
|
||||
echo --- Modules list
|
||||
module list
|
||||
echo --- End listings
|
||||
|
||||
#
|
||||
# Versions should be consistent with setup script
|
||||
#
|
||||
|
||||
# GCC
|
||||
# Dates at https://gcc.gnu.org/releases.html
|
||||
gcc_vnew=9.2.0 # 2019-08-12
|
||||
gcc_vold=7.3.0 # 2018-01-25
|
||||
|
||||
#For Intel:
|
||||
gcc_vintel=7.4.0 # 2018-12-06
|
||||
|
||||
#PGI 19.4
|
||||
# makelocalrc configured with 8.3.0 currently
|
||||
gcc_vpgi=8.3.0 # 2019-02-22
|
||||
|
||||
# For CUDA toolkit compatibility
|
||||
gcc_vcuda=8.3.0 # 2019-02-22
|
||||
|
||||
# LLVM
|
||||
# Dates at http://releases.llvm.org/
|
||||
llvm_vnew=9.0.0 # 2019-09-19
|
||||
llvm_vold=5.0.1 # 2017-12-21
|
||||
# for CUDA 10.1 update 2
|
||||
llvm_vcuda=8.0.0 # 2019-03-
|
||||
|
||||
# HDF5
|
||||
hdf5_vnew=1.10.5 # Releeased 2019-02-28
|
||||
hdf5_vold=1.8.19 # Released 2017-06-16
|
||||
|
||||
# CMake
|
||||
# Dates at https://cmake.org/files/
|
||||
cmake_vnew=3.16.2 # Released 2019-12-19
|
||||
cmake_vold=3.10.2 # Released 2018-01-18
|
||||
|
||||
# OpenMPI
|
||||
# Dates at https://www.open-mpi.org/software/ompi/v4.0/
|
||||
ompi_vnew=4.0.2 # Released 2019-10-07
|
||||
ompi_vold=2.1.2 # Released 2017-09-20
|
||||
|
||||
libxml2_vnew=2.9.9 # Released 2019-01-03 See http://xmlsoft.org/sources/
|
||||
libxml2_vold=2.9.1 # Released 2013-04-19
|
||||
|
||||
# FFTW
|
||||
# Dates at http://www.fftw.org/release-notes.html
|
||||
fftw_vnew=3.3.8 # Released 2018-05-28
|
||||
fftw_vold=3.3.4 # Released 2014-03-16
|
||||
|
||||
# BOOST
|
||||
# Dates at https://www.boost.org/users/history/
|
||||
boost_vnew=1.70.0 # Released 2019-04-12
|
||||
boost_vold=1.65.1 # Released 2016-05-13
|
||||
|
||||
echo --- START env `date`
|
||||
echo --- gcc@${gcc_vold}
|
||||
spack install gcc@${gcc_vold}
|
||||
spack load gcc@${gcc_vold}
|
||||
spack compiler find
|
||||
echo --- Spack compilers
|
||||
spack compilers
|
||||
|
||||
if [[ $1 == "clean" ]]; then
|
||||
cat >$HOME/.spack/packages.yaml<<EOF
|
||||
packages:
|
||||
all:
|
||||
compiler: [gcc@${gcc_vold}]
|
||||
EOF
|
||||
fi
|
||||
|
||||
spack install --no-checksum libxml2@${libxml2_vold}%gcc@${gcc_vold}
|
||||
spack install cmake@${cmake_vold}%gcc@${gcc_vold}
|
||||
spack install boost@${boost_vold}%gcc@${gcc_vold}
|
||||
spack install openmpi@${ompi_vold}%gcc@${gcc_vold}
|
||||
spack install hdf5@${hdf5_vold}~mpi %gcc@${gcc_vold}
|
||||
spack install fftw@${fftw_vold}%gcc@${gcc_vold}
|
||||
spack unload gcc@${gcc_vold}
|
||||
echo --- gcc@${gcc_vnew}
|
||||
spack install gcc@${gcc_vnew}
|
||||
spack load gcc@${gcc_vnew}
|
||||
spack compiler find
|
||||
spack install --no-checksum libxml2@${libxml2_vnew}%gcc@${gcc_vnew}
|
||||
#spack install --no-checksum cmake@${cmake_vnew}%gcc@${gcc_vnew}
|
||||
# BUILD LATEST CMAKE WITH OLD GCC DUE TO BUILD FAILURE WITH GCC_VNEW
|
||||
spack install --no-checksum cmake@${cmake_vnew}%gcc@${gcc_vold}
|
||||
spack install boost@${boost_vnew}%gcc@${gcc_vnew}
|
||||
spack install openmpi@${ompi_vnew}%gcc@${gcc_vnew}^libxml2@${libxml2_vnew}%gcc@${gcc_vnew}
|
||||
#spack install hdf5@${hdf5_vnew}%gcc@${gcc_vnew}^openmpi@${ompi_vnew}%gcc@${gcc_vnew}
|
||||
spack install hdf5@${hdf5_vnew}~mpi %gcc@${gcc_vnew}
|
||||
spack install fftw@${fftw_vnew}%gcc@${gcc_vnew}
|
||||
spack unload gcc@${gcc_vnew}
|
||||
echo --- llvm@${llvm_vnew}
|
||||
spack install llvm@${llvm_vnew}
|
||||
spack load llvm@${llvm_vnew}
|
||||
spack compiler find
|
||||
spack unload llvm@${llvm_vnew}
|
||||
|
||||
# before you can build openmpi you need to define a fortran compiler for clang
|
||||
# we do this with a python script since we need to modify relatively free form yaml.
|
||||
# which btw we assume is in the same directory
|
||||
spack python ./add_fortran_for_spack_clang.py gcc@${gcc_vnew}
|
||||
spack install openmpi@${ompi_vnew}%clang@${llvm_vnew}^libxml2@${libxml2_vnew}%gcc@${gcc_vnew}
|
||||
|
||||
echo --- Convenience
|
||||
spack install git
|
||||
echo --- Python setup for NEXUS
|
||||
spack install py-numpy@1.18.0%gcc@${gcc_vnew}
|
||||
spack install py-h5py^py-numpy@1.18.0%gcc@${gcc_vnew}
|
||||
spack install py-pandas@0.25.1^py-numpy@1.18.0%gcc@${gcc_vnew}
|
||||
spack install py-scipy@1.4.1^py-numpy@1.18.0%gcc@${gcc_vnew}
|
||||
spack activate py-numpy@1.18.0%gcc@${gcc_vnew}
|
||||
spack activate py-h5py^py-numpy@1.18.0%gcc@${gcc_vnew}
|
||||
spack activate py-pandas@0.25.1^py-numpy@1.18.0%gcc@${gcc_vnew}
|
||||
spack activate py-scipy@1.4.1^py-numpy@1.18.0%gcc@${gcc_vnew}
|
||||
|
||||
echo --- FINISH `date`
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash
|
||||
|
||||
. $SPACK_ROOT/share/spack/setup-env.sh
|
||||
|
||||
. ../tests/test_automation/spack_supported_package_versions.sh
|
||||
|
||||
spack load gcc@$gcc_vnew%gcc@$gcc_vold
|
||||
spack load boost@$boost_vnew%gcc@$gcc_vnew
|
||||
spack load hdf5@$hdf5_vnew%gcc@$gcc_vnew~mpi
|
||||
spack load openssl@1.1.1d%gcc@9.2.0+systemcerts
|
||||
spack load cmake@$cmake_vnew%gcc@$gcc_vnew
|
||||
spack load hwloc%gcc@$gcc_vnew
|
||||
spack load libiconv%gcc@$gcc_vnew
|
||||
spack load openmpi@$ompi_vnew%gcc@$gcc_vnew
|
||||
spack load libxml2@$libxml2_vnew%gcc@$gcc_vnew
|
||||
spack load fftw@$fftw_vnew%gcc@$gcc_vnew
|
||||
spack load openblas%gcc@$gcc_vnew
|
||||
spack load netlib-lapack%gcc@$gcc_vnew
|
||||
|
||||
#if you've installed more than one python for the new compiler this will fail
|
||||
spack load python%gcc@$gcc_vnew
|
||||
|
||||
QMC_IMMUTABLE_FLAGS="-DBUILD_AFQMC=1"
|
||||
|
||||
echo ${PATH}
|
Loading…
Reference in New Issue