qmcpack/CMake/ctest_script.cmake

376 lines
13 KiB
CMake

# ctest script for building, running, and submitting the test results
# Usage: ctest -s script,build
# build = debug / optimized / valgrind / coverage
# Note: this test will use use the number of processors defined in the variable N_PROCS,
# the environment variables
# N_PROCS, or the number of processors available (if not specified)
# N_PROCS_BUILD, or N_PROCS (if not specified)
# N_CONCURRENT_TESTS, or N_PROCS (if not specified)
# TEST_SITE_NAME, or HOSTNAME (if not specified)
# Get the source directory based on the current directory
IF ( NOT DEFINED QMC_SOURCE_DIR )
SET( QMC_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/.." )
ENDIF()
# Check that we specified the build type to run
IF ( NOT DEFINED CTEST_BUILD_NAME )
SET( CTEST_BUILD_NAME "$ENV{QMCPACK_TEST_SUBMIT_NAME}" )
ENDIF()
IF ( NOT DEFINED CTEST_BUILD_NAME )
SET( CTEST_BUILD_NAME "QMCPACK_TEST_SUBMIT_NAME-unset" )
ENDIF()
IF( NOT DEFINED CTEST_SCRIPT_ARG )
MESSAGE(FATAL_ERROR "No build specified: ctest -S /path/to/script,build (debug/optimized/valgrind")
ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "debug" )
SET( CMAKE_BUILD_TYPE "Debug" )
SET( USE_VALGRIND FALSE )
ELSEIF( (${CTEST_SCRIPT_ARG} STREQUAL "optimized") OR (${CTEST_SCRIPT_ARG} STREQUAL "opt") OR (${CTEST_SCRIPT_ARG} STREQUAL "release") )
SET( CMAKE_BUILD_TYPE "Release" )
SET( CTEST_COVERAGE_COMMAND )
SET( ENABLE_GCOV "false" )
SET( USE_VALGRIND FALSE )
ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "valgrind" )
SET( CMAKE_BUILD_TYPE "Debug" )
SET( CTEST_COVERAGE_COMMAND )
SET( ENABLE_GCOV "false" )
SET( USE_VALGRIND TRUE )
ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "coverage" )
SET( CMAKE_BUILD_TYPE "Debug" )
SET( CTEST_COVERAGE_COMMAND "gcov" )
SET( ENABLE_GCOV "true" )
SET( CTEST_BUILD_NAME "${CTEST_BUILD_NAME}-coverage" )
ELSE()
MESSAGE(FATAL_ERROR "Invalid build (${CTEST_SCRIPT_ARG}): ctest -S /path/to/script,build (debug/opt/valgrind")
ENDIF()
# Set the number of processors
IF( NOT DEFINED N_PROCS )
SET( N_PROCS $ENV{N_PROCS} )
ENDIF()
IF( NOT DEFINED N_PROCS )
SET(N_PROCS 1)
# Linux:
SET(cpuinfo_file "/proc/cpuinfo")
IF(EXISTS "${cpuinfo_file}")
FILE(STRINGS "${cpuinfo_file}" procs REGEX "^processor.: [0-9]+$")
list(LENGTH procs N_PROCS)
ENDIF()
# Mac:
IF(APPLE)
find_program(cmd_sys_pro "system_profiler")
if(cmd_sys_pro)
execute_process(COMMAND ${cmd_sys_pro} OUTPUT_VARIABLE info)
STRING(REGEX REPLACE "^.*Total Number of Cores: ([0-9]+).*$" "\\1" N_PROCS "${info}")
ENDIF()
ENDIF()
# Windows:
IF(WIN32)
SET(N_PROCS "$ENV{NUMBER_OF_PROCESSORS}")
ENDIF()
ENDIF()
# Set the number of processors for compilation and running tests
IF( NOT DEFINED N_PROCS_BUILD )
IF(DEFINED ENV{N_PROCS_BUILD})
SET( N_PROCS_BUILD $ENV{N_PROCS_BUILD} )
ELSE()
SET( N_PROCS_BUILD ${N_PROCS} )
ENDIF()
ENDIF()
IF( NOT DEFINED N_CONCURRENT_TESTS )
IF( DEFINED ENV{N_CONCURRENT_TESTS} )
SET( N_CONCURRENT_TESTS $ENV{N_CONCURRENT_TESTS} )
ELSE()
SET( N_CONCURRENT_TESTS ${N_PROCS} )
ENDIF()
ENDIF()
MESSAGE("Testing with ${N_CONCURRENT_TESTS} processors")
# Set basic variables
SET( CTEST_PROJECT_NAME "QMCPACK" )
SET( CTEST_SOURCE_DIRECTORY "${QMC_SOURCE_DIR}" )
SET( CTEST_BINARY_DIRECTORY "." )
SET( CTEST_DASHBOARD "Nightly" )
SET( CTEST_TEST_TIMEOUT 900 )
SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS 500 )
SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 500 )
SET( CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 100000 )
SET( CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 100000 )
SET( NIGHTLY_START_TIME "18:00:00 EST" )
SET( CTEST_NIGHTLY_START_TIME "22:00:00 EST" )
SET( CTEST_COMMAND "\"${CTEST_EXECUTABLE_NAME}\" -D ${CTEST_DASHBOARD}" )
SET( CTEST_USE_LAUNCHERS TRUE )
SET( CTEST_CMAKE_GENERATOR "Unix Makefiles")
IF ( NOT DEFINED MAKE_CMD )
SET( MAKE_CMD make )
ENDIF()
IF ( BUILD_SERIAL )
SET( CTEST_BUILD_COMMAND "${MAKE_CMD} -i" )
ELSE ( BUILD_SERIAL )
SET( CTEST_BUILD_COMMAND "${MAKE_CMD} -i -j ${N_PROCS_BUILD}" )
MESSAGE("Building with ${N_PROCS_BUILD} processors")
ENDIF( BUILD_SERIAL )
# Set valgrind options
IF ( USE_VALGRIND )
SET( VALGRIND_COMMAND_OPTIONS "--tool=memcheck --leak-check=yes --track-fds=yes --num-callers=50 --show-reachable=yes --suppressions=${QMC_SOURCE_DIR}/src/ValgrindSuppresionFile" )
SET( MEMORYCHECK_COMMAND ${VALGRIND_COMMAND} )
SET( MEMORYCHECKCOMMAND ${VALGRIND_COMMAND} )
SET( CTEST_MEMORYCHECK_COMMAND ${VALGRIND_COMMAND} )
SET( CTEST_MEMORYCHECKCOMMAND ${VALGRIND_COMMAND} )
SET( CTEST_MEMORYCHECK_COMMAND_OPTIONS ${VALGRIND_COMMAND_OPTIONS} )
SET( CTEST_MEMORYCHECKCOMMAND_OPTIONS ${VALGRIND_COMMAND_OPTIONS} )
ENDIF()
# Clear the binary directory and create an initial cache
CTEST_EMPTY_BINARY_DIRECTORY( ${CTEST_BINARY_DIRECTORY} )
FILE(WRITE "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" "CTEST_TEST_CTEST:BOOL=1")
# Set the configure options
SET( CTEST_OPTIONS )
IF ( NOT DEFINED CMAKE_TOOLCHAIN_FILE )
IF ( DEFINED CMAKE_C_COMPILER )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}" )
ENDIF()
IF ( DEFINED CMAKE_CXX_COMPILER )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" )
ENDIF()
IF ( DEFINED CMAKE_C_FLAGS )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_FLAGS='${CMAKE_C_FLAGS}'" )
ENDIF()
IF ( DEFINED CMAKE_CXX_FLAGS )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_CXX_FLAGS='${CMAKE_CXX_FLAGS}'" )
ENDIF()
ELSE()
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_TOOLCHAIN_FILE='${CMAKE_TOOLCHAIN_FILE}'" )
ENDIF()
IF ( DEFINED ENABLE_GCOV )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_GCOV:BOOL=${ENABLE_GCOV}" )
ENDIF()
IF ( DEFINED CMAKE_BUILD_TYPE )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" )
ENDIF()
IF ( DEFINED QMC_DATA )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQMC_DATA='${QMC_DATA}'" )
ENDIF()
IF ( DEFINED QE_BIN )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQE_BIN='${QE_BIN}'" )
ENDIF()
IF ( DEFINED QMC_CUDA )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQMC_CUDA=${QMC_CUDA}" )
ENDIF()
IF ( DEFINED ENABLE_CUDA )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_CUDA=${ENABLE_CUDA}" )
ENDIF()
IF ( DEFINED QMC_COMPLEX )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQMC_COMPLEX=${QMC_COMPLEX}" )
ENDIF()
IF ( DEFINED QMC_MPI )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQMC_MPI=${QMC_MPI}" )
ENDIF()
IF ( DEFINED MPIEXEC_EXECUTABLE )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE}" )
ENDIF()
IF ( DEFINED MPIEXEC_NUMPROC_FLAG )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG}" )
ENDIF()
IF ( DEFINED MPIEXEC_PREFLAGS )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS}" )
ENDIF()
IF ( DEFINED QMC_MIXED_PRECISION )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQMC_MIXED_PRECISION=${QMC_MIXED_PRECISION}" )
ENDIF()
IF ( DEFINED CUDA_ARCH )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCUDA_ARCH='${CUDA_ARCH}'" )
ENDIF()
IF ( DEFINED BUILD_AFQMC )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DBUILD_AFQMC=${BUILD_AFQMC}" )
ENDIF()
IF ( DEFINED BLA_VENDOR )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DBLA_VENDOR='${BLA_VENDOR}'" )
ENDIF()
IF ( DEFINED ENABLE_TIMERS )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_TIMERS=${ENABLE_TIMERS}" )
ENDIF()
IF ( DEFINED CMAKE_PREFIX_PATH )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_PREFIX_PATH='${CMAKE_PREFIX_PATH}'" )
ENDIF()
IF ( DEFINED QMC_EXTRA_LIBS )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQMC_EXTRA_LIBS:STRING='${QMC_EXTRA_LIBS}'" )
ENDIF()
IF ( DEFINED QMC_OPTIONS )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};${QMC_OPTIONS}" )
ENDIF()
IF ( DEFINED HDF5_ROOT )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DHDF5_ROOT='${HDF5_ROOT}'" )
ENDIF()
MESSAGE("Configure options:")
MESSAGE(" ${CTEST_OPTIONS}")
IF ( NOT DEFINED CTEST_SITE )
SET( CTEST_SITE $ENV{TEST_SITE_NAME} )
ENDIF()
IF ( NOT DEFINED CTEST_SITE )
SITE_NAME( HOSTNAME )
SET( CTEST_SITE ${HOSTNAME} )
ENDIF()
# Configure and run the tests
CTEST_START("${CTEST_DASHBOARD}")
CTEST_UPDATE()
CTEST_CONFIGURE(
BUILD ${CTEST_BINARY_DIRECTORY}
SOURCE ${CTEST_SOURCE_DIRECTORY}
OPTIONS "${CTEST_OPTIONS}"
)
# Run the configure, build and tests
CTEST_BUILD()
# Submit the results to oblivion
SET( CTEST_DROP_METHOD "https" )
SET( CTEST_DROP_SITE "cdash.qmcpack.org" )
SET( CTEST_DROP_LOCATION "/CDash/submit.php?project=QMCPACK" )
SET( CTEST_DROP_SITE_CDASH TRUE )
SET( DROP_SITE_CDASH TRUE )
CTEST_SUBMIT( PARTS Configure Build )
IF ( USE_VALGRIND )
CTEST_MEMCHECK( EXCLUDE procs PARALLEL_LEVEL ${N_PROCS} )
CTEST_SUBMIT( PARTS MemCheck )
ELSEIF (CTEST_COVERAGE_COMMAND)
# Skip the normal tests when doing coverage
ELSE()
# CTEST_TEST( INCLUDE short PARALLEL_LEVEL ${N_PROCS} )
# run and submit the classified tests to their corresponding track
CTEST_START( "${CTEST_DASHBOARD}" TRACK "Deterministic" APPEND)
CTEST_TEST( INCLUDE_LABEL "deterministic" PARALLEL_LEVEL ${N_CONCURRENT_TESTS} )
CTEST_SUBMIT( PARTS Test )
CTEST_START( "${CTEST_DASHBOARD}" TRACK "Converter" APPEND)
CTEST_TEST( INCLUDE_LABEL "converter" PARALLEL_LEVEL ${N_CONCURRENT_TESTS} )
CTEST_SUBMIT( PARTS Test )
CTEST_START( "${CTEST_DASHBOARD}" TRACK "Performance" APPEND)
CTEST_TEST( INCLUDE_LABEL "performance" PARALLEL_LEVEL 16 )
CTEST_SUBMIT( PARTS Test )
# run and submit unclassified tests to the default track
CTEST_START( "${CTEST_DASHBOARD}" TRACK "${CTEST_DASHBOARD}" APPEND)
CTEST_TEST( EXCLUDE_LABEL "deterministic|performance|converter|unstable" PARALLEL_LEVEL ${N_CONCURRENT_TESTS} )
CTEST_SUBMIT( PARTS Test )
# Only the result checking is placed in the unstable category and parent process must run before it
# To fullfil this implicit dependency, the unstable category is placed last.
# A better solution is needed to make the dependency explicit.
CTEST_START( "${CTEST_DASHBOARD}" TRACK "Unstable" APPEND)
CTEST_TEST( INCLUDE_LABEL "unstable" PARALLEL_LEVEL ${N_CONCURRENT_TESTS} )
CTEST_SUBMIT( PARTS Test )
ENDIF()
IF( CTEST_COVERAGE_COMMAND )
# Path prefix to remove to shorten some file names. The final SRC_ROOT Should not contain '..'
GET_FILENAME_COMPONENT(SRC_ROOT2 ${QMC_SOURCE_DIR} DIRECTORY)
GET_FILENAME_COMPONENT(SRC_ROOT ${SRC_ROOT2} DIRECTORY)
EXECUTE_PROCESS(COMMAND "pwd" OUTPUT_VARIABLE CURRENT_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
#MESSAGE("Using new code coverage path in ${CTEST_SOURCE_DIRECTORY}")
#MESSAGE("Using new code coverage path bin: ${CTEST_BINARY_DIRECTORY}")
INCLUDE("${CTEST_SOURCE_DIRECTORY}/CMake/compareGCOV.cmake")
# Base test
CLEAR_GCDA(${CTEST_BINARY_DIRECTORY})
CTEST_TEST(INCLUDE_LABEL coverage)
FILE(REMOVE_RECURSE ${CTEST_BINARY_DIRECTORY}/tgcov_base_raw)
GENERATE_GCOV(${CTEST_BINARY_DIRECTORY} ${CTEST_BINARY_DIRECTORY}/tgcov_base_raw "USE_LONG_FILE_NAMES" ${SRC_ROOT})
FILTER_GCOV(${CTEST_BINARY_DIRECTORY}/tgcov_base_raw)
FILE(REMOVE_RECURSE ${CTEST_BINARY_DIRECTORY}/tgcov_base)
MERGE_GCOV(${CTEST_BINARY_DIRECTORY}/tgcov_base_raw ${CTEST_BINARY_DIRECTORY}/tgcov_base ${SRC_ROOT})
# Generate gcov
CLEAR_GCDA(${CTEST_BINARY_DIRECTORY})
# Remove gcda files
CTEST_TEST(INCLUDE_LABEL unit)
# Generate gcov
FILE(REMOVE_RECURSE ${CTEST_BINARY_DIRECTORY}/tgcov_unit_raw)
GENERATE_GCOV(${CTEST_BINARY_DIRECTORY} ${CTEST_BINARY_DIRECTORY}/tgcov_unit_raw "USE_LONG_FILE_NAMES" ${SRC_ROOT})
FILTER_GCOV(${CTEST_BINARY_DIRECTORY}/tgcov_unit_raw)
FILE(REMOVE_RECURSE ${CTEST_BINARY_DIRECTORY}/tgcov_unit)
MERGE_GCOV(${CTEST_BINARY_DIRECTORY}/tgcov_unit_raw ${CTEST_BINARY_DIRECTORY}/tgcov_unit ${SRC_ROOT})
# Generate diff
FILE(REMOVE_RECURSE ${CTEST_BINARY_DIRECTORY}/tgcov_diff)
COMPARE_GCOV(${CTEST_BINARY_DIRECTORY}/tgcov_base
${CTEST_BINARY_DIRECTORY}/tgcov_unit
${CTEST_BINARY_DIRECTORY}/tgcov_diff
tgcov_diff)
# create tar file
CREATE_GCOV_TAR(${CTEST_BINARY_DIRECTORY} tgcov_unit)
CREATE_GCOV_TAR(${CTEST_BINARY_DIRECTORY} tgcov_base)
FILE(GLOB DIFF_GCOV_FILES ${CTEST_BINARY_DIRECTORY}/tgcov_diff/*.gcov)
SET( CTEST_BUILD_NAME_ORIGINAL "${CTEST_BUILD_NAME}")
SET( CTEST_BUILD_NAME "${CTEST_BUILD_NAME_ORIGINAL}-diff" )
CTEST_START(${CTEST_DASHBOARD})
IF(EXISTS "${CTEST_BINARY_DIRECTORY}/gcov.tar")
MESSAGE("submitting ${CTEST_BINARY_DIRECTORY}/gcov.tar")
CTEST_SUBMIT(CDASH_UPLOAD "${CTEST_BINARY_DIRECTORY}/gcov.tar"
CDASH_UPLOAD_TYPE GcovTar)
ENDIF()
SET( CTEST_BUILD_NAME "${CTEST_BUILD_NAME_ORIGINAL}-base" )
CTEST_START(${CTEST_DASHBOARD})
IF(EXISTS "${CTEST_BINARY_DIRECTORY}/gcov_tgcov_base.tar")
MESSAGE("submitting ${CTEST_BINARY_DIRECTORY}/gcov_tgcov_base.tar")
CTEST_SUBMIT(CDASH_UPLOAD "${CTEST_BINARY_DIRECTORY}/gcov_tgcov_base.tar"
CDASH_UPLOAD_TYPE GcovTar)
ENDIF()
SET( CTEST_BUILD_NAME "${CTEST_BUILD_NAME_ORIGINAL}-unit" )
CTEST_START(${CTEST_DASHBOARD})
IF(EXISTS "${CTEST_BINARY_DIRECTORY}/gcov_tgcov_unit.tar")
MESSAGE("submitting ${CTEST_BINARY_DIRECTORY}/gcov_tgcov_unit.tar")
CTEST_SUBMIT(CDASH_UPLOAD "${CTEST_BINARY_DIRECTORY}/gcov_tgcov_unit.tar"
CDASH_UPLOAD_TYPE GcovTar)
ENDIF()
ENDIF()
# Clean up
# exec_program("make distclean")