From a754c71bfcac32295d19a7cd00b8040eb5de795e Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 16 Jun 2026 15:07:31 +0200 Subject: [PATCH 1/3] Supported CMAKE_BUILD_TYPE=PROFILE --- build_tsmp2.sh | 56 ++++++++++++++++++++++++-------------- cmake/BuildOASIS3MCT.cmake | 14 ++++++++-- env/jsc.2026.gnu.openmpi | 26 +++++++++++++----- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index 9d07bc3c..fe5aebc6 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -36,7 +36,7 @@ function help_tsmp2() { echo " --OASIS_SRC Set OASIS3-MCT directory" echo " --PDAF_SRC Set PDAF_SRC directory" echo " --no_update Skip component model download" - echo " --build_type Set build configuration: 'DEBUG' 'RELEASE'" + echo " --build_type Set build configuration: 'DEBUG', 'RELEASE', or 'PROFILE'" echo " --build_dir Set build dir cmake, if not set bld/_ is used. Build artifacts will be generated in this folder." echo " --install_dir Set install dir cmake, if not set bin/_ is used. Model executables and libraries will be installed here" echo " --clean_first Delete build_dir if it already exists" @@ -207,8 +207,19 @@ if [ "${update_compsrc}" != n ]; then dwn_compsrc clm35 clm35_src "CLM3.5" fi +# 2. Determine build configuration +if [[ -z "$build_type" ]];then + build_type="RELEASE" +fi +if [[ ${build_type^^} == "DEBUG" || ${build_type^^} == "RELEASE" || ${build_type^^} == "PROFILE" ]]; then + cmake_build_type="${build_type^^}" +else + echo "ABORT: Unsupported build_type=${build_type}" + exit 1 +fi + # -# 2. Source environment file +# 3. Source environment file # if [[ ! -z "${env}" ]]; then @@ -230,15 +241,22 @@ if [[ ! -f "${env}" ]]; then exit 1 fi + if [[ -n "${env}" ]]; then message "Sourcing environment..." - # TODO: Fix this GPU thing on another PR + # TODO: Env file parameterization is temporary and should be fixed if [[ "$parflowGPU" == "y" ]];then - source "${env}" --parflowgpu - else - source "${env}" + env_params="--parflowgpu" + fi + if [[ ${build_type^^} == "PROFILE" ]];then + env_params+="${env_params} --profile" + if [[ ":${TSMP2_ENV_FILE}:" != *"jsc.2026.gnu.openmpi"* ]]; then + message "ERROR: Profiling builds are only currently supported on jsc.2026.gnu.openmpi." + exit 1 + fi fi + source "${env}" "${env_params}" fi # TSMP2_ENV_FILE should be set either (1) through --env, (2) as a shell variable, @@ -249,9 +267,14 @@ if [[ -z "${TSMP2_ENV_FILE}" ]]; then fi # -# 3. Set CMake build directory +# 4. Set CMake build directory # -BUILD_ID="${SYSTEMNAME^^}_${model_id}" +if [[ ${build_type^^} == "RELEASE" ]]; then + BUILD_ID="${SYSTEMNAME^^}_${model_id}" +else + BUILD_ID="${SYSTEMNAME^^}_${build_type^^}_${model_id}" +fi + if [[ -z "${build_dir}" ]]; then cmake_build_dir="${cmake_tsmp2_dir}/bld/${BUILD_ID}" else @@ -293,18 +316,9 @@ fi build_log="$(dirname ${cmake_build_dir})/${BUILD_ID}_$(date +%Y-%m-%d_%H-%M).log" # -# 4. Set the rest of the CMake options +# 5. Set the rest of the CMake options # message "Setting CMAKE options..." -if [[ -z "$build_type" ]];then - build_type="RELEASE" -fi -if [[ ${build_type^^} == "DEBUG" || ${build_type^^} == "RELEASE" ]]; then - cmake_build_type="${build_type^^}" -else - echo "ABORT: Unsupported build_type=${build_type}" - exit 1 -fi if [ -z "${verbose_makefile}" ]; then cmake_verbose_makefile="OFF" @@ -320,7 +334,7 @@ fi mkdir -p "${cmake_install_dir}" # -# 5. CMake configure +# 6. CMake configure # message "" message "====================" @@ -344,7 +358,7 @@ cmake ${cmake_conf} |& tee -a "${build_log}" message "== CMAKE GENERATE PROJECT finished" # -# 6. CMake build and install +# 7. CMake build and install # message "CMAKE build:" message "cmake --build ${cmake_build_dir} |& tee -a $build_log" @@ -359,7 +373,7 @@ cmake --install ${cmake_build_dir} |& tee -a $build_log message "== CMAKE INSTALL finished" # -# 7. Post-installation steps +# 8. Post-installation steps # message "Copying build log and environment file to ${cmake_install_dir}..." if [[ -n "${env}" ]]; then diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index c66840fd..25c6aeb6 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -1,12 +1,22 @@ find_package(NetCDF REQUIRED) find_package(OpenMP REQUIRED) +set(OASIS_C_COMPILER ${CMAKE_C_COMPILER}) +set(OASIS_Fortran_COMPILER ${CMAKE_Fortran_COMPILER}) + if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") set(OPTIM "-g") set(MCT_DEBUGFLAG "--enable-debugging") elseif (CMAKE_BUILD_TYPE STREQUAL "RELEASE") set(OPTIM "-O2") set(MCT_DEBUGFLAG "") +elseif (CMAKE_BUILD_TYPE STREQUAL "PROFILE") + # TODO: which build flags to use for profiling? + set(OPTIM "-O2") + set(MCT_DEBUGFLAG "") + # Score-P MPI wrappers have problems with OpenMP; bypass it for now + set(OASIS_C_COMPILER mpicc) + set(OASIS_Fortran_COMPILER mpif90) else() # Assume CMAKE_BUILD_TYPE=RELEASE if CMAKE_BUILD_TYPE is unknown set(OPTIM "-O2") @@ -29,8 +39,8 @@ file(APPEND ${OASIS_MAKE_INC} "F90 = ${CMAKE_Fortran_COMPILER}\n") file(APPEND ${OASIS_MAKE_INC} "F = $(F90)\n") file(APPEND ${OASIS_MAKE_INC} "f90 = $(F90)\n") file(APPEND ${OASIS_MAKE_INC} "f = $(F90)\n") -file(APPEND ${OASIS_MAKE_INC} "CC = ${CMAKE_C_COMPILER}\n") -file(APPEND ${OASIS_MAKE_INC} "LD = ${CMAKE_Fortran_COMPILER} -L${MPI_Fortran_LIB_DIR}\n") +file(APPEND ${OASIS_MAKE_INC} "CC = ${OASIS_C_COMPILER}\n") +file(APPEND ${OASIS_MAKE_INC} "LD = ${OASIS_Fortran_COMPILER} -L${MPI_Fortran_LIB_DIR}\n") file(APPEND ${OASIS_MAKE_INC} "AR = ar\n") file(APPEND ${OASIS_MAKE_INC} "ARFLAGS = -ruv\n") file(APPEND ${OASIS_MAKE_INC} "DYNOPT = -fPIC\n") diff --git a/env/jsc.2026.gnu.openmpi b/env/jsc.2026.gnu.openmpi index d006c930..7a4a419b 100644 --- a/env/jsc.2026.gnu.openmpi +++ b/env/jsc.2026.gnu.openmpi @@ -52,6 +52,12 @@ fi module load ecCodes # Set default MPI compilers +if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then + module load Scalasca + export PROFILE_CC=scorep-mpicc + export PROFILE_FC=scorep-mpif90 + export PROFILE_CXX=scorep-mpicxx +fi export CC=mpicc export FC=mpif90 export CXX=mpicxx @@ -64,14 +70,20 @@ export SUNDIALS_ROOT=$EBROOTSUNDIALS # Display compiler settings module list echo "=============== COMPILER SETTINGS ===============" -echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" -echo " MPI lib: OpenMPI v$EBVERSIONOPENMPI" -echo " C: $($CC --version | head -n 1)" -echo " C++: $($CXX --version | head -n 1)" -echo " Fortran: $($FC --version | head -n 1)" +echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" +echo " MPI lib: OpenMPI v$EBVERSIONOPENMPI" +echo " C: $($CC --version | head -n 1)" +echo " C++: $($CXX --version | head -n 1)" +echo " Fortran: $($FC --version | head -n 1)" if [[ "$1" == "--parflowgpu" ]]; then - echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" - echo " CUDAARCHS: $CUDAARCHS" + echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" + echo " CUDAARCHS: $CUDAARCHS" +fi +if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then + echo " PROFILE_CC: ${PROFILE_CC}" + echo " PROFILE_FC: ${PROFILE_FC}" + echo "PROFILE_CXX: ${PROFILE_CXX}" + echo "PROFILE_VER: ${EBVERSIONSCALASCA}" fi echo "==================================================" echo "" From a2394fd528d29bce3a0ad51ffee87748cbbcb648 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 16 Jun 2026 16:23:09 +0200 Subject: [PATCH 2/3] Set profiling compilers on eCLM and Parflow if CMAKE_BUILD_TYPE=PROFILE --- build_tsmp2.sh | 9 +++++---- cmake/BuildOASIS3MCT.cmake | 16 +++------------- cmake/BuildParFlow.cmake | 17 ++++++++++++++--- cmake/BuildeCLM.cmake | 13 +++++++++++-- env/jsc.2026.gnu.openmpi | 21 +++++++++------------ 5 files changed, 42 insertions(+), 34 deletions(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index fe5aebc6..10026374 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -247,16 +247,17 @@ if [[ -n "${env}" ]]; then # TODO: Env file parameterization is temporary and should be fixed if [[ "$parflowGPU" == "y" ]];then - env_params="--parflowgpu" + env_gpu_params="--parflowgpu" fi if [[ ${build_type^^} == "PROFILE" ]];then - env_params+="${env_params} --profile" + env_profile_params="--profile" if [[ ":${TSMP2_ENV_FILE}:" != *"jsc.2026.gnu.openmpi"* ]]; then message "ERROR: Profiling builds are only currently supported on jsc.2026.gnu.openmpi." exit 1 - fi + fi fi - source "${env}" "${env_params}" + message "SOURCE command: source ${env} ${env_gpu_params} ${env_profile_params}" + source "${env}" ${env_gpu_params} ${env_profile_params} fi # TSMP2_ENV_FILE should be set either (1) through --env, (2) as a shell variable, diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index 25c6aeb6..f71ec295 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -1,22 +1,12 @@ find_package(NetCDF REQUIRED) find_package(OpenMP REQUIRED) -set(OASIS_C_COMPILER ${CMAKE_C_COMPILER}) -set(OASIS_Fortran_COMPILER ${CMAKE_Fortran_COMPILER}) - if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") set(OPTIM "-g") set(MCT_DEBUGFLAG "--enable-debugging") -elseif (CMAKE_BUILD_TYPE STREQUAL "RELEASE") - set(OPTIM "-O2") - set(MCT_DEBUGFLAG "") -elseif (CMAKE_BUILD_TYPE STREQUAL "PROFILE") - # TODO: which build flags to use for profiling? +elseif (CMAKE_BUILD_TYPE STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE STREQUAL "PROFILE") set(OPTIM "-O2") set(MCT_DEBUGFLAG "") - # Score-P MPI wrappers have problems with OpenMP; bypass it for now - set(OASIS_C_COMPILER mpicc) - set(OASIS_Fortran_COMPILER mpif90) else() # Assume CMAKE_BUILD_TYPE=RELEASE if CMAKE_BUILD_TYPE is unknown set(OPTIM "-O2") @@ -39,8 +29,8 @@ file(APPEND ${OASIS_MAKE_INC} "F90 = ${CMAKE_Fortran_COMPILER}\n") file(APPEND ${OASIS_MAKE_INC} "F = $(F90)\n") file(APPEND ${OASIS_MAKE_INC} "f90 = $(F90)\n") file(APPEND ${OASIS_MAKE_INC} "f = $(F90)\n") -file(APPEND ${OASIS_MAKE_INC} "CC = ${OASIS_C_COMPILER}\n") -file(APPEND ${OASIS_MAKE_INC} "LD = ${OASIS_Fortran_COMPILER} -L${MPI_Fortran_LIB_DIR}\n") +file(APPEND ${OASIS_MAKE_INC} "CC = ${CMAKE_C_COMPILER}\n") +file(APPEND ${OASIS_MAKE_INC} "LD = ${CMAKE_Fortran_COMPILER} -L${MPI_Fortran_LIB_DIR}\n") file(APPEND ${OASIS_MAKE_INC} "AR = ar\n") file(APPEND ${OASIS_MAKE_INC} "ARFLAGS = -ruv\n") file(APPEND ${OASIS_MAKE_INC} "DYNOPT = -fPIC\n") diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 81711453..7a7ac7ca 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -75,14 +75,25 @@ if (DEFINED ENV{SUNDIALS_ROOT} AND NOT ${PDAF}) set(PF_SUNDIALS_FLAG "-DSUNDIALS_ROOT=$ENV{SUNDIALS_ROOT}") endif() +# Profiling build (Experimental) +if (CMAKE_BUILD_TYPE STREQUAL "PROFILE") + set(PF_C_COMPILER $ENV{TSMP2_PROFILE_CC}) + set(PF_CXX_COMPILER $ENV{TSMP2_PROFILE_CXX}) + set(PF_Fortran_COMPILER $ENV{TSMP2_PROFILE_FC}) +else() + set(PF_C_COMPILER ${CMAKE_C_COMPILER}) + set(PF_CXX_COMPILER ${CMAKE_CXX_COMPILER}) + set(PF_Fortran_COMPILER ${CMAKE_Fortran_COMPILER}) +endif() + # Pass options to ParFlow CMake ExternalProject_Add(ParFlow PREFIX ParFlow SOURCE_DIR ${PARFLOW_SRC} - CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + CMAKE_ARGS -DCMAKE_C_COMPILER=${PF_C_COMPILER} -DCMAKE_C_FLAGS=${PF_CFLAGS} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} + -DCMAKE_CXX_COMPILER=${PF_CXX_COMPILER} + -DCMAKE_Fortran_COMPILER=${PF_Fortran_COMPILER} -DCMAKE_Fortran_FLAGS=${PF_FFLAGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} diff --git a/cmake/BuildeCLM.cmake b/cmake/BuildeCLM.cmake index 42f0f642..f54c7d3a 100644 --- a/cmake/BuildeCLM.cmake +++ b/cmake/BuildeCLM.cmake @@ -14,12 +14,21 @@ if(${PDAF}) list(APPEND PDAF_FLAGS -DUSE_PDAF=True) endif() +# Experimental +if (CMAKE_BUILD_TYPE STREQUAL "PROFILE") + set(eCLM_C_COMPILER $ENV{TSMP2_PROFILE_CC}) + set(eCLM_Fortran_COMPILER $ENV{TSMP2_PROFILE_FC}) +else() + set(eCLM_C_COMPILER ${CMAKE_C_COMPILER}) + set(eCLM_Fortran_COMPILER ${CMAKE_Fortran_COMPILER}) +endif() + ExternalProject_Add(eCLM PREFIX eCLM SOURCE_DIR ${eCLM_SRC} SOURCE_SUBDIR src - CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} + CMAKE_ARGS -DCMAKE_C_COMPILER=${eCLM_C_COMPILER} + -DCMAKE_Fortran_COMPILER=${eCLM_Fortran_COMPILER} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_PREFIX_PATH=${OASIS_ROOT} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} diff --git a/env/jsc.2026.gnu.openmpi b/env/jsc.2026.gnu.openmpi index 7a4a419b..5f8bb02d 100644 --- a/env/jsc.2026.gnu.openmpi +++ b/env/jsc.2026.gnu.openmpi @@ -52,11 +52,11 @@ fi module load ecCodes # Set default MPI compilers -if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then +if [[ "$2" == "--profile" ]]; then module load Scalasca - export PROFILE_CC=scorep-mpicc - export PROFILE_FC=scorep-mpif90 - export PROFILE_CXX=scorep-mpicxx + export TSMP2_PROFILE_CC=$(which scorep-mpicc) + export TSMP2_PROFILE_FC=$(which scorep-mpif90) + export TSMP2_PROFILE_CXX=$(which scorep-mpicxx) fi export CC=mpicc export FC=mpif90 @@ -72,18 +72,15 @@ module list echo "=============== COMPILER SETTINGS ===============" echo " Machine: ${SYSTEMNAME} on Stages/$STAGE" echo " MPI lib: OpenMPI v$EBVERSIONOPENMPI" -echo " C: $($CC --version | head -n 1)" -echo " C++: $($CXX --version | head -n 1)" -echo " Fortran: $($FC --version | head -n 1)" +echo " C: ${CC} (GCC ${EBVERSIONGCCCORE})" +echo " C++: ${CXX} (GCC ${EBVERSIONGCCCORE})" +echo " Fortran: ${FC} (GCC ${EBVERSIONGCCCORE})" if [[ "$1" == "--parflowgpu" ]]; then echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" echo " CUDAARCHS: $CUDAARCHS" fi -if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then - echo " PROFILE_CC: ${PROFILE_CC}" - echo " PROFILE_FC: ${PROFILE_FC}" - echo "PROFILE_CXX: ${PROFILE_CXX}" - echo "PROFILE_VER: ${EBVERSIONSCALASCA}" +if [[ "$2" == "--profile" ]]; then + echo " Profiler: Scalasca v${EBVERSIONSCALASCA}" fi echo "==================================================" echo "" From 519e7e0069d769f2b0c77ddc5c561d9a376a6de0 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 17 Jun 2026 08:34:03 +0200 Subject: [PATCH 3/3] Ignored compiler warnings when compiling ParFlow with scorep-mpicc --- cmake/BuildParFlow.cmake | 5 ++++- env/jsc.2026.gnu.openmpi | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 7a7ac7ca..f0c9eb8f 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -41,7 +41,10 @@ endif() # Set compiler flags if(CMAKE_C_COMPILER_ID STREQUAL "GNU") # Flags were based from https://github.com/parflow/parflow/blob/c8aa8d7140db19153194728b8fa9136b95177b6d/.github/workflows/linux.yml#L486 - set(PF_CFLAGS "-Wall -Werror -Wno-unused-result -Wno-unused-function -Wno-stringop-overread") + set(PF_CFLAGS "-Wall -Wno-unused-result -Wno-unused-function -Wno-stringop-overread") + if (NOT CMAKE_BUILD_TYPE STREQUAL "PROFILE") + string(APPEND PF_CFLAGS " -Werror") #compile warnings not ignored for DEBUG and RELEASE builds + endif() if(${PDAF}) # parflow-pdaf fork currently ignores unused variables string(APPEND PF_CFLAGS " -Wno-unused-variable") diff --git a/env/jsc.2026.gnu.openmpi b/env/jsc.2026.gnu.openmpi index 5f8bb02d..9b472ab8 100644 --- a/env/jsc.2026.gnu.openmpi +++ b/env/jsc.2026.gnu.openmpi @@ -52,7 +52,7 @@ fi module load ecCodes # Set default MPI compilers -if [[ "$2" == "--profile" ]]; then +if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then module load Scalasca export TSMP2_PROFILE_CC=$(which scorep-mpicc) export TSMP2_PROFILE_FC=$(which scorep-mpif90) @@ -79,7 +79,7 @@ if [[ "$1" == "--parflowgpu" ]]; then echo " nvcc: $(nvcc --version | tail -n 1 | cut -d" " -f2)" echo " CUDAARCHS: $CUDAARCHS" fi -if [[ "$2" == "--profile" ]]; then +if [[ "$1" == "--profile" || "$2" == "--profile" ]]; then echo " Profiler: Scalasca v${EBVERSIONSCALASCA}" fi echo "=================================================="