From cabe36b5464a76c776c296e835278b69ec53fd79 Mon Sep 17 00:00:00 2001 From: lparisi Date: Tue, 11 Nov 2025 09:13:31 +0000 Subject: [PATCH 01/18] Added cirrus-ex tests for lammps(ethanol),pytorch(deepcam) and castep --- tests/apps/castep/castep_check.py | 15 +++++-- tests/apps/lammps/ethanol.py | 41 ++++++++++++++++--- tests/libs/pytorch/deepcam.py | 38 ++++++++++++++--- tests/libs/pytorch/mlperf_build.py | 8 +++- .../src/build_pytorch_cirrus_ex_env.sh | 30 ++++++++++++++ 5 files changed, 117 insertions(+), 15 deletions(-) create mode 100644 tests/libs/pytorch/src/build_pytorch_cirrus_ex_env.sh diff --git a/tests/apps/castep/castep_check.py b/tests/apps/castep/castep_check.py index 2fc24c13..c3f4652a 100644 --- a/tests/apps/castep/castep_check.py +++ b/tests/apps/castep/castep_check.py @@ -52,7 +52,7 @@ def extract_calctime(self): class CASTEPCPUCheck(CASTEPBaseCheck): """CASTEP Check for CPU""" - valid_systems = ["archer2:compute", "cirrus:compute"] + valid_systems = ["archer2:compute", "cirrus:compute","cirrus-ex:compute"] descr = "CASTEP corrctness and performance test" executable_opts = ["al3x3"] @@ -62,12 +62,16 @@ class CASTEPCPUCheck(CASTEPBaseCheck): reference["archer2:compute"] = {} reference["archer2:compute"]["calctime"] = (126, -0.1, 0.1, "s") reference["archer2:compute"]["runtime"] = (132, -0.1, 0.1, "s") - + reference["cirrus:compute"] = {} - reference["cirrus:compute"]["calctime"] = (325.9, -0.1, 0.1, "s") reference["cirrus:compute"]["runtime"] = (328.2, -0.1, 0.1, "s") + reference["cirrus-ex:compute"] = {} + reference["cirrus-ex:compute"]["runtime"] = (82, -0.1, 0.1, "s") + reference["cirrus-ex:compute"]["calctime"] = (74, -0.1, 0.1, "s") + + @run_after("init") def setup_environment(self): """Setup environment""" @@ -75,6 +79,11 @@ def setup_environment(self): self.modules = ["castep"] self.num_tasks = 512 self.num_tasks_per_node = 128 + + if self.current_system.name in ["cirrus-ex"]: + self.modules = ["castep"] + self.num_tasks = 288 * 2 + self.num_tasks_per_node = 288 if self.current_system.name in ["cirrus"]: self.modules = ["castep/22.1.1"] diff --git a/tests/apps/lammps/ethanol.py b/tests/apps/lammps/ethanol.py index 0e718926..2167cf31 100644 --- a/tests/apps/lammps/ethanol.py +++ b/tests/apps/lammps/ethanol.py @@ -34,6 +34,7 @@ class LAMMPSBaseEthanol(LAMMPSBase): reference = { "cirrus:compute-gpu": {"energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol")}, "archer2:compute": {"energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol")}, + "cirrus-ex:compute": {"energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol")}, "archer2-tds:compute": {"energy": (ethanol_energy_reference, -0.01, 0.01, "kJ/mol")}, } @@ -49,25 +50,28 @@ def extract_energy(self): ) -@rfm.simple_test class LAMMPSEthanolCPU(LAMMPSBaseEthanol): """ReFrame LAMMPS Ethanol test for performance checks""" - valid_systems = ["archer2:compute"] descr = LAMMPSBaseEthanol.descr + " -- CPU" - stream_binary = fixture(BuildLAMMPS, scope="environment") - + reference["archer2-tds:compute"] = {} reference["archer2:compute"] = {} + reference["cirrus-ex:compute"] = {} reference["archer2:compute"]["performance"] = (11.250, -0.05, None, "ns/day") - reference["archer2-tds:compute"]["performance"] = (11.250, -0.05, None, "ns/day") + reference["cirrus-ex:compute"]["performance"] = (27.56, -0.05, None, "ns/day") + reference["archer2-tds:compute"]["performance"] = (11.250, -0.05, None, "ns/day") + @run_after("init") def setup_nnodes(self): """sets up number of tasks per node""" if self.current_system.name in ["archer2"]: self.num_tasks_per_node = 128 + else: + if self.current_system.name in ["cirrus-ex"]: + self.num_tasks_per_node = 288 @run_after("setup") def set_executable(self): @@ -77,8 +81,33 @@ def set_executable(self): @run_before("run") def setup_resources(self): """sets up number of tasks""" - self.num_tasks = self.n_nodes * self.cores.get(self.current_partition.fullname, 1) + self.num_tasks = self.n_nodes * self.num_tasks_per_node + +@rfm.simple_test +class LAMMPSEthanolCPURunReframeBuild(LAMMPSEthanolCPU): + """ReFrame LAMMPS Ethanol test for performance checks of the reframe-built executable""" + + valid_systems = ["archer2:compute"] + descr = LAMMPSEthanolCPU.descr + ", reframe-bult-executable -- CPU" + stream_binary = fixture(BuildLAMMPS, scope="environment") + @run_after("setup") + def set_executable(self): + """sets up executable""" + self.executable = os.path.join(self.stream_binary.build_system.builddir, "lmp") + + +@rfm.simple_test +class LAMMPSEthanolCPURunModule(LAMMPSEthanolCPU): + """ReFrame LAMMPS Ethanol test for performance checks of the default lammps module""" + + valid_systems = ["archer2:compute","cirrus-ex:compute"] + descr = LAMMPSEthanolCPU.descr + ", default lammps module -- CPU" + + @run_after("setup") + def set_executable(self): + """sets up executable""" + self.executable = "lmp" @rfm.simple_test class LAMMPSEthanolGPU(LAMMPSBaseEthanol): diff --git a/tests/libs/pytorch/deepcam.py b/tests/libs/pytorch/deepcam.py index 3716ebb6..8f077cf9 100644 --- a/tests/libs/pytorch/deepcam.py +++ b/tests/libs/pytorch/deepcam.py @@ -44,6 +44,7 @@ def setup_job(self): "qos": {"qos": "gpu"}, "gpu": {"num_gpus_per_node": str(self.num_gpus)}, } + self.executable_opts = [ train_script, "--wireup_method nccl-slurm", @@ -77,32 +78,59 @@ def add_additional_options(self): @rfm.simple_test class DeepCAMCPUtest(DeepCAMBase): """Running DeepCAM on CPU""" - + num_nodes = 4 num_tasks_per_node = 8 num_cpus_per_task = 16 num_tasks = 32 time_limit = "1h" - valid_systems = ["archer2:compute"] + valid_systems = ["archer2:compute","cirrus-ex:compute"] valid_prog_environs = ["PrgEnv-cray"] - extra_resources = {"qos": {"qos": "standard"}} - reference = {"archer2:compute": {"epoch-time": (272, -0.1, 0.1, "s")}} + reference = {"archer2:compute": {"epoch-time": (272, -0.1, 0.1, "s")} + ,"cirrus-ex:compute": {"epoch-time": (119.7, -0.01, 0.01, "s")} + } + + @run_after("init") + def set_modules(self): + if self.current_system.name in ["cirrus-ex"]: + self.modules = ["py-torch"] + @run_after("init") + def set_num_tasks(self): + + if self.current_system.name in ["cirrus-ex"]: + self.num_tasks_per_node = 24 + self.num_cpus_per_task = 12 + + if self.current_system.name in ["archer2"]: + self.num_tasks_per_node = 8 + self.num_cpus_per_task = 16 + + self.num_tasks = self.num_nodes * self.num_tasks_per_node + @run_after("setup") def setup_job(self): """Set-up submission script""" train_script = os.path.join(self.mlperf_benchmarks.stagedir, "hpc", "deepcam", "src", "deepCam", "train.py") + + data_dir_prefix = None + if self.current_system.name in ["cirrus-ex"]: + data_dir_prefix = "/work/z04/shared/mlperf-hpc/deepcam/mini/" + if self.current_system.name in ["archer2"]: + data_dir_prefix = "/work/z19/shared/mlperf-hpc/deepcam/mini/" + self.executable_opts = [ train_script, "--wireup_method mpi", "--run_tag reframe-cpu", "--output_dir ${JOB_OUTPUT_PATH}", - "--data_dir_prefix /work/z19/shared/mlperf-hpc/deepcam/mini", + f"--data_dir_prefix {data_dir_prefix}", "--local_batch_size 1", "--max_inter_threads ${SLURM_CPUS_PER_TASK}", "--max_epochs 5", "--seed ${SLURM_JOB_ID}", ] + self.env_vars = { "OMP_NUM_THREADS": "${SLURM_CPUS_PER_TASK}", "HOME": "${HOME/home/work}", diff --git a/tests/libs/pytorch/mlperf_build.py b/tests/libs/pytorch/mlperf_build.py index f64390de..db251be9 100644 --- a/tests/libs/pytorch/mlperf_build.py +++ b/tests/libs/pytorch/mlperf_build.py @@ -45,11 +45,17 @@ def prepare_env(self): if part in ("archer2:compute-gpu", "cirrus:compute-gpu"): self.modules = ["pytorch/1.13.1-gpu"] self.env_vars["PYVENV_NAME"] = "reframe-mlperf-gpu" + build_script = "build_pytorch_env.sh" elif part == "archer2:compute": self.modules = ["pytorch/1.13.0a0"] self.env_vars["PYVENV_NAME"] = "reframe-mlperf-cpu" + elif part == "cirrus-ex:compute": + self.modules = ["py-torch"] + self.env_vars["PYVENV_NAME"] = "reframe-mlperf-cpu" + build_script = "build_pytorch_cirrus_ex_env.sh" + - self.build_system.commands = ["chmod u+x build_pytorch_env.sh", "./build_pytorch_env.sh"] + self.build_system.commands = [f"chmod u+x {build_script}", f"./{build_script}"] @sanity_function def sanity_check_build(self): diff --git a/tests/libs/pytorch/src/build_pytorch_cirrus_ex_env.sh b/tests/libs/pytorch/src/build_pytorch_cirrus_ex_env.sh new file mode 100644 index 00000000..09c1f04a --- /dev/null +++ b/tests/libs/pytorch/src/build_pytorch_cirrus_ex_env.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +PYTHON_TAG=python`echo ${CRAY_PYTHON_LEVEL} | cut -d. -f1-2` + +# PRFX=${HOME/home/work}/pyenvs +# PYVENV_ROOT=${PRFX}/${PYVENV_NAME} +# PYVENV_SITEPKGS=${PYVENV_ROOT}/lib/${PYTHON_TAG}/site-packages + +echo ${PYVENV_NAME} + +mkdir -p ${PYVENV_NAME} +cd ${PYVENV_NAME} + +python -m venv --system-site-packages ${PYVENV_NAME} +extend-venv-activate ${PYVENV_NAME} +source ${PYVENV_NAME}/bin/activate + +mkdir -p ${PYVENV_NAME}/repos +cd ${PYVENV_NAME}/repos + +## Install mlperf loging package +wget https://github.com/mlcommons/logging/archive/refs/tags/5.1.0-rc4.tar.gz +tar -zxvf 5.1.0-rc4.tar.gz +python -m pip install -e logging-5.1.0-rc4 + +python -m pip install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git + +python -m pip install h5py + +deactivate \ No newline at end of file From 3c6210b1473c4aeaaedfb9f85ed46ed188694c16 Mon Sep 17 00:00:00 2001 From: lparisi Date: Tue, 11 Nov 2025 11:07:06 +0000 Subject: [PATCH 02/18] Enabling porting openfoam to cirrus-ex --- tests/apps/openfoam/openfoam_org_base.py | 2 +- tests/apps/openfoam/openfoam_org_dambreak.py | 54 ++++++++++++++++---- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/tests/apps/openfoam/openfoam_org_base.py b/tests/apps/openfoam/openfoam_org_base.py index 8a5b062c..56d53e86 100644 --- a/tests/apps/openfoam/openfoam_org_base.py +++ b/tests/apps/openfoam/openfoam_org_base.py @@ -10,7 +10,7 @@ class OpenFOAMBase(rfm.RunOnlyRegressionTest): v_major = "10" v_patch = "20230119" version = f"{v_major}.{v_patch}" - valid_systems = ["archer2:compute"] + valid_systems = ["archer2:compute","cirrus-ex:compute"] valid_prog_environs = ["PrgEnv-gnu"] maintainers = ["e.broadway@epcc.ed.ac.uk", "j.richings@epcc.ed.ac.uk"] diff --git a/tests/apps/openfoam/openfoam_org_dambreak.py b/tests/apps/openfoam/openfoam_org_dambreak.py index 9f506123..7df152e6 100644 --- a/tests/apps/openfoam/openfoam_org_dambreak.py +++ b/tests/apps/openfoam/openfoam_org_dambreak.py @@ -12,13 +12,16 @@ class OpenFOAMDamBreakBase(OpenFOAMBase): """OpenFOAM DamBreak test base class""" num_tasks_per_node = 1 - num_cpus_per_task = 128 time_limit = "10m" - freq = parameter(["2250000", "2000000"]) + + @run_after("init") def setup_params(self): """sets up extra parameters""" + + if self.current_system.name in ["archer2"]: + freq = parameter(["2250000", "2000000"]) if self.current_system.name in ["archer2"]: self.env_vars = { "OMP_NUM_THREADS": str(self.num_cpus_per_task), @@ -26,6 +29,18 @@ def setup_params(self): "SLURM_CPU_FREQ_REQ": self.freq, } + @run_before("run") + def set_num_tasks(self): + """Sets number of tasks""" + if self.current_system.name in ["archer2"]: + self.num_cpus_per_task = 128 + self.num_tasks_per_node = 1 + self.num_tasks = self.num_tasks_per_node * self.num_nodes + elif self.current_system.name in ["cirrus-ex"]: + self.num_cpus_per_task = 288 + self.num_tasks_per_node = 1 + self.num_tasks = self.num_tasks_per_node * self.num_nodes + @run_before("run") def setup_testcase(self): """set up test case""" @@ -44,31 +59,42 @@ def set_reference(self): """Changes reference values""" if self.current_system.name in ["archer2"]: # https://reframe-hpc.readthedocs.io/en/stable/utility_functions_reference.html#reframe.utility.ScopedDict - self.reference["archer2:compute:performance"] = self.reference_performance[self.freq] - + self.reference["archer2:compute:performance"] = self.reference_performance_archer2[self.freq] + + elif self.current_system.name in ["cirrus-ex"]: + self.reference["cirrus-ex:compute:performance"] = self.reference_performance_cirrus_ex + class OpenFOAMDamBreakOneNode(OpenFOAMDamBreakBase): """OpenFOAM DamBreak base test on 1 node""" executable = "interFoam" executable_opts = ("").split() - modules = [f"openfoam/org/v{OpenFOAMBase.version}"] num_tasks = 1 + num_nodes = 1 - reference_performance = { + reference_performance_archer2 = { "2000000": (6, -0.1, 0.1, "seconds"), "2250000": (3.6, -0.1, 0.1, "seconds"), } + reference_performance_cirrus_ex = (5, -0.5, 0.5, "seconds") + @rfm.simple_test class OpenFOAMDamBreakOneNodeModule(OpenFOAMDamBreakOneNode): """OpenFOAM DamBreak test on 1 node with module""" executable = "interFoam" - modules = [f"openfoam/org/v{OpenFOAMBase.version}"] + @run_before("run") + def load_modules(self): + if self.current_system.name in ["archer2"]: + self.modules = [f"openfoam/org/v{OpenFOAMBase.version}"] + elif self.current_system.name in ["cirrus-ex"]: + self.modules = [f"openfoam-org"] + @rfm.simple_test class OpenFOAMDamBreakOneNodeBuild(OpenFOAMDamBreakOneNode): @@ -98,11 +124,13 @@ class OpenFOAMDamBreakParallel(OpenFOAMDamBreakBase): num_tasks = 4 executable_opts = ("-parallel").split() - reference_performance = { + reference_performance_archer2 = { "2000000": (5, -0.5, 0.5, "seconds"), "2250000": (5, -0.5, 0.5, "seconds"), } + reference_performance_cirrus_ex = (5, -0.5, 0.5, "seconds") + @run_before("run") def setup_testcase(self): """Set up test case""" @@ -118,9 +146,15 @@ def assert_finished_parallel(self): @rfm.simple_test class OpenFOAMDamBreakParallelModule(OpenFOAMDamBreakParallel): """OpenFOAM DamBreak test on 4 nodes with module""" - + executable = "interFoam" - modules = [f"openfoam/org/v{OpenFOAMBase.version}"] + @run_before("run") + def load_modules(self): + if self.current_system.name in ["archer2"]: + modules = [f"openfoam/org/v{OpenFOAMBase.version}"] + elif self.current_system.name in ["cirrus-ex"]: + modules = [f"openfoam-org"] + @rfm.simple_test From d077c9d68084bfadf12dd6897ac90b3294feb829 Mon Sep 17 00:00:00 2001 From: lparisi Date: Thu, 8 Jan 2026 17:29:56 +0000 Subject: [PATCH 03/18] Adapted openfoam-org tutorial for version 12 --- tests/apps/openfoam/openfoam_org_base.py | 5 ++- tests/apps/openfoam/openfoam_org_dambreak.py | 36 ++++++++++++++++---- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/tests/apps/openfoam/openfoam_org_base.py b/tests/apps/openfoam/openfoam_org_base.py index 56d53e86..2c1c4df4 100644 --- a/tests/apps/openfoam/openfoam_org_base.py +++ b/tests/apps/openfoam/openfoam_org_base.py @@ -6,11 +6,10 @@ class OpenFOAMBase(rfm.RunOnlyRegressionTest): """ReFrame OpenFOAM test base class""" - - v_major = "10" + + v_major = "12" v_patch = "20230119" version = f"{v_major}.{v_patch}" - valid_systems = ["archer2:compute","cirrus-ex:compute"] valid_prog_environs = ["PrgEnv-gnu"] maintainers = ["e.broadway@epcc.ed.ac.uk", "j.richings@epcc.ed.ac.uk"] diff --git a/tests/apps/openfoam/openfoam_org_dambreak.py b/tests/apps/openfoam/openfoam_org_dambreak.py index 7df152e6..f0de2912 100644 --- a/tests/apps/openfoam/openfoam_org_dambreak.py +++ b/tests/apps/openfoam/openfoam_org_dambreak.py @@ -8,11 +8,13 @@ from openfoam_org_build import CompileOpenFOAM + class OpenFOAMDamBreakBase(OpenFOAMBase): """OpenFOAM DamBreak test base class""" num_tasks_per_node = 1 time_limit = "10m" + valid_systems = ["archer2:compute"] @@ -28,6 +30,12 @@ def setup_params(self): "OMP_PLACES": "cores", "SLURM_CPU_FREQ_REQ": self.freq, } + + if self.current_system.name in ["cirrus-ex"]: + self.env_vars.update( + {"FOAM_INSTALL_DIR": "$FOAM_ETC/.."} + ) + @run_before("run") def set_num_tasks(self): @@ -44,9 +52,19 @@ def set_num_tasks(self): @run_before("run") def setup_testcase(self): """set up test case""" + + if (self.v_major == "10"): + tutorial_sub_dir="tutorials/multiphase/interFoam/laminar/damBreak/damBreak" + else: + if (self.v_major == "12"): + tutorial_sub_dir="tutorials/incompressibleVoF/damBreak" + else: + raise ValueError("Unsupported OpenFOAM version") + + self.prerun_cmds = [ "source ${FOAM_INSTALL_DIR}/etc/bashrc", - "cp -r ${FOAM_INSTALL_DIR}/tutorials/multiphase/interFoam/laminar/damBreak/damBreak .", + f"cp -r $FOAM_INSTALL_DIR/{tutorial_sub_dir} .", "cd damBreak", "blockMesh", "cp 0/alpha.water.orig 0/alpha.water", @@ -71,6 +89,7 @@ class OpenFOAMDamBreakOneNode(OpenFOAMDamBreakBase): executable = "interFoam" executable_opts = ("").split() + num_tasks = 1 num_nodes = 1 @@ -79,12 +98,13 @@ class OpenFOAMDamBreakOneNode(OpenFOAMDamBreakBase): "2250000": (3.6, -0.1, 0.1, "seconds"), } - reference_performance_cirrus_ex = (5, -0.5, 0.5, "seconds") + reference_performance_cirrus_ex = (3, -0.5, 0.5, "seconds") @rfm.simple_test class OpenFOAMDamBreakOneNodeModule(OpenFOAMDamBreakOneNode): """OpenFOAM DamBreak test on 1 node with module""" + valid_systems = ["archer2:compute","cirrus-ex:compute"] executable = "interFoam" @@ -121,7 +141,7 @@ def set_executable(self): class OpenFOAMDamBreakParallel(OpenFOAMDamBreakBase): """OpenFOAM DamBreak base test on 4 nodes""" - num_tasks = 4 + num_nodes = 4 executable_opts = ("-parallel").split() reference_performance_archer2 = { @@ -135,7 +155,9 @@ class OpenFOAMDamBreakParallel(OpenFOAMDamBreakBase): def setup_testcase(self): """Set up test case""" super().setup_testcase() - self.prerun_cmds = [*self.prerun_cmds, "decomposePar"] + self.prerun_cmds = [*self.prerun_cmds, + "cp -r ../decomposeParDict system", + "decomposePar"] @sanity_function def assert_finished_parallel(self): @@ -147,13 +169,15 @@ def assert_finished_parallel(self): class OpenFOAMDamBreakParallelModule(OpenFOAMDamBreakParallel): """OpenFOAM DamBreak test on 4 nodes with module""" + valid_systems = ["archer2:compute","cirrus-ex:compute"] + executable = "interFoam" @run_before("run") def load_modules(self): if self.current_system.name in ["archer2"]: - modules = [f"openfoam/org/v{OpenFOAMBase.version}"] + self.modules = [f"openfoam/org/v{OpenFOAMBase.version}"] elif self.current_system.name in ["cirrus-ex"]: - modules = [f"openfoam-org"] + self.modules = [f"openfoam-org"] From 357231b6c71aa3e24c1ce16a6c21f607700a313c Mon Sep 17 00:00:00 2001 From: lparisi Date: Fri, 9 Jan 2026 14:19:39 +0000 Subject: [PATCH 04/18] Added regression tests for cp2k --- tests/apps/cp2k/cp2k_check.py | 185 ++++++++++++++++++++++------------ 1 file changed, 120 insertions(+), 65 deletions(-) diff --git a/tests/apps/cp2k/cp2k_check.py b/tests/apps/cp2k/cp2k_check.py index 00737b9b..b9b84df5 100644 --- a/tests/apps/cp2k/cp2k_check.py +++ b/tests/apps/cp2k/cp2k_check.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""ReFrame test for cp2k""" +"""ReFrame tests for cp2k""" import reframe as rfm import reframe.utility.sanity as sn +from reframe.core.backends import register_launcher +from reframe.core.launchers import JobLauncher class CP2KBaseCheck(rfm.RunOnlyRegressionTest): @@ -15,62 +17,19 @@ class CP2KBaseCheck(rfm.RunOnlyRegressionTest): executable = "cp2k.psmp" # Additional Slurm parameters. Requires adding to config file first. extra_resources = {"qos": {"qos": "standard"}} - # Output files to be retained - keep_files = ["cp2k.out"] - - # Set time limit - - time_limit = "20m" - + maintainers = ["j.richings@epcc.ed.ac.uk"] use_multithreading = False - tags = {"applications", "performance"} - - # Reference value to validate run with - energy_reference = -870.934788 - - reference = { - "*": {"energy": (energy_reference, -0.01, 0.01, "a.u.")}, - "cirrus:compute": {"performance": (1300, -0.05, 0.05, "seconds")}, - } - - reference_performance = { - "2000000": (350, -0.1, 0.1, "seconds"), - "2250000": (250, -0.1, 0.1, "seconds"), - } - - @sanity_function - def assert_finished(self): - """Sanity check that simulation finished successfully""" - return sn.assert_found("CP2K ", self.keep_files[0]) - - @performance_function("a.u.", perf_key="energy") - def extract_energy(self): - """Extract value of system energy for performance check""" - return sn.extractsingle( - r"ENERGY\| Total FORCE_EVAL \( QS \) energy \[a.u.\]:\s+(?P\S+)", - self.keep_files[0], - "energy", - float, - ) - - @performance_function("seconds", perf_key="performance") - def extract_perf(self): - """Extract performance value to compare with reference value""" - return sn.extractsingle( - r"\s+CP2K(?:\s+\d+\.?\d*){5}\s+(?P\S+)", - self.keep_files[0], - "perf", - float, - ) - + tags = {"applications"} @rfm.simple_test -class CP2KARCHER2(CP2KBaseCheck): - """CP2K test""" - +class CP2KARCHER2HFX(CP2KBaseCheck): + """CP2K performance test""" + # Select system to use valid_systems = ["archer2:compute"] + # Output files to be retained + keep_files = ["cp2k.out"] # Set Programming Environment valid_prog_environs = ["PrgEnv-gnu", "PrgEnv-gnu-hf"] # Description of test @@ -84,21 +43,27 @@ class CP2KARCHER2(CP2KBaseCheck): num_tasks_per_node = 16 num_cpus_per_task = 8 time_limit = "10m" - + # Reference value to validate run with + energy_reference = -870.934788 + tags = CP2KBaseCheck.tags.union({"performance"}) reference_performance = { "2000000": (350, -0.1, 0.1, "seconds"), "2250000": (250, -0.1, 0.1, "seconds"), } + @sanity_function + def assert_finished(self): + """Sanity check that simulation finished successfully""" + return sn.assert_found("CP2K ", self.keep_files[0]) + @run_after("init") def setup_params(self): """sets up extra parameters""" # self.descr += self.freq if self.current_system.name in ["archer2"]: - self.env_vars = {"OMP_NUM_THREADS": str(self.num_cpus_per_task), "OMP_PLACES": "cores"} - - # "SLURM_CPU_FREQ_REQ": self.freq, - # } + self.env_vars = {"OMP_NUM_THREADS": str(self.num_cpus_per_task), + "OMP_PLACES": "cores" + } @run_before("performance") def set_reference(self): @@ -108,27 +73,117 @@ def set_reference(self): self.reference["archer2:compute:performance"] = self.reference_performance[ "2250000" if self.current_environ.name[-3:] == "-hf" else "2000000" ] + reference = { + "*": {"energy": (energy_reference, -0.01, 0.01, "a.u.")} + } + + reference_performance = { + "2000000": (350, -0.1, 0.1, "seconds"), + "2250000": (250, -0.1, 0.1, "seconds"), + } + + @performance_function("a.u.", perf_key="energy") + def extract_energy(self): + """Extract value of system energy for performance check""" + return sn.extractsingle( + r"ENERGY\| Total FORCE_EVAL \( QS \) energy \[a.u.\]:\s+(?P\S+)", + self.keep_files[0], + "energy", + float, + ) + + @performance_function("seconds", perf_key="performance") + def extract_perf(self): + """Extract performance value to compare with reference value""" + return sn.extractsingle( + r"\s+CP2K(?:\s+\d+\.?\d*){5}\s+(?P\S+)", + self.keep_files[0], + "perf", + float, + ) + + +@rfm.simple_test +class FetchCP2K(rfm.RunOnlyRegressionTest): + """ + Fetch CP2K source code, which contains the regression tests and benchmarks. + """ + + descr = "Fetch cp2k code" + version = variable(str, value="2025.2") + executable = "wget" + executable_opts = [f"https://github.com/cp2k/cp2k/archive/refs/tags/v{version}.tar.gz"] + local = True + valid_systems = ["cirrus-ex:login"] + valid_prog_environs = ["*"] + @sanity_function + def validate_download(self): + """Validate the download was successful""" + return sn.assert_eq(self.job.exitcode, 0) @rfm.simple_test -class CP2KCPUCirrus(CP2KBaseCheck): - """CP2K test for Cirrus""" +class CP2KCPUCirrusExRegressionTests(CP2KBaseCheck): + """ + CP2K regression tests for cirrus-ex + + This runs the CP2K tests from the CP2K regression test suite. + They are good functionality tests but not very useful for performance. + + """ # Select system to use - valid_systems = ["cirrus:compute"] + valid_systems = ["cirrus-ex:compute"] # Set Programming Environment - valid_prog_environs = ["gcc"] + valid_prog_environs = ["PrgEnv-gnu"] # Description of test - descr = "CP2K test" + descr = "CP2K regression tests" + launcher="cp2k_reg_tests" + # Command line options for executable - executable_opts = ("-i input_bulk_HFX_3.inp -o cp2k.out ").split() + + executable = "cp2k.psmp" + + executable_opts = ["-v"] + # slurm parameters - num_tasks = 360 - num_tasks_per_node = 18 + num_tasks = 144 + num_tasks_per_node = 144 num_cpus_per_task = 2 - time_limit = "1h" + time_limit = "20m" + + cp2k_src = fixture(FetchCP2K, scope="environment") env_vars = { "OMP_NUM_THREADS": str(num_cpus_per_task), "OMP_PLACES": "cores", + "CP2K_APP" : "$(which cp2k.psmp)", + "CP2K_DIR" : "${CP2K_APP::-10}" } + + @sanity_function + def assert_all_tests_completed(self): + """Sanity check that simulation finished successfully""" + return sn.assert_found("Status: OK",self.stdout) + + @run_before("run") + def launch_reg_tests(self): + """ + The command to launch the regression tests is a python script executed serially ( not in parallel). + + In this implementation we use pre-run commands. An alternative is to use a custom launcher. However this needs to be specified in the config, in a custom programming environment. I think that solution is worse , because it pollutes a configuration file with test specific logic. + + """ + + source_tar_file=f"v{self.cp2k_src.version}.tar.gz" + + self.prerun_cmds = [ + f"cp {self.cp2k_src.stagedir}/{source_tar_file} .", + f"tar -zxf {source_tar_file}", + f"cp2k-{self.cp2k_src.version}/tests/do_regtest.py \ + --workbasedir=$(pwd) \ + --maxtasks=72 \ + --mpiranks=2 \ + --ompthreads=${{OMP_NUM_THREADS}} \ + --mpiexec=\"srun --ntasks=2 --cpus-per-task=${{OMP_NUM_THREADS}}\" \ + $CP2K_DIR psmp" ] \ No newline at end of file From 44bcdf69c9d10f48afa7534f6819b5efdc027306 Mon Sep 17 00:00:00 2001 From: lparisi Date: Fri, 9 Jan 2026 14:56:01 +0000 Subject: [PATCH 05/18] Adapted OpenFoam to run v12 on cirrus-ex and v10 on Archer2 --- tests/apps/openfoam/openfoam_org_base.py | 23 +++++++++-- tests/apps/openfoam/openfoam_org_build.py | 27 ++++++++----- tests/apps/openfoam/openfoam_org_dambreak.py | 18 +++++---- tests/apps/openfoam/src/decomposeParDict | 42 ++++++++++++++++++++ 4 files changed, 87 insertions(+), 23 deletions(-) create mode 100644 tests/apps/openfoam/src/decomposeParDict diff --git a/tests/apps/openfoam/openfoam_org_base.py b/tests/apps/openfoam/openfoam_org_base.py index 2c1c4df4..e7b669f7 100644 --- a/tests/apps/openfoam/openfoam_org_base.py +++ b/tests/apps/openfoam/openfoam_org_base.py @@ -7,11 +7,9 @@ class OpenFOAMBase(rfm.RunOnlyRegressionTest): """ReFrame OpenFOAM test base class""" - v_major = "12" - v_patch = "20230119" - version = f"{v_major}.{v_patch}" + valid_prog_environs = ["PrgEnv-gnu"] - + maintainers = ["e.broadway@epcc.ed.ac.uk", "j.richings@epcc.ed.ac.uk"] use_multithreading = False tags = {"applications", "performance"} @@ -30,3 +28,20 @@ def extract_perf(self): "time", float, ) + @run_after("init") + def set_version_vars(self): + """sets up version variables""" + + if self.current_system.name in ["archer2"]: + self.v_major = "10" + self.v_patch = "20230119" + if self.current_system.name in ["cirrus-ex"]: + self.v_major = "12" + self.v_patch = "0" + else: + raise ValueError(f"OpenFoam version for System {self.current_system.name} not recognised .") + self.version = f"{self.v_major}-{self.v_patch}" + + + + diff --git a/tests/apps/openfoam/openfoam_org_build.py b/tests/apps/openfoam/openfoam_org_build.py index 9a06a7fe..a1167c71 100644 --- a/tests/apps/openfoam/openfoam_org_build.py +++ b/tests/apps/openfoam/openfoam_org_build.py @@ -13,23 +13,28 @@ class FetchOpenFOAM(rfm.RunOnlyRegressionTest): valid_systems = ["archer2:login"] executable = "bash" - executable_opts = [ - "-c", - f""" - wget -O OpenFOAM-{OpenFOAMBase.v_major}-{OpenFOAMBase.v_patch}.tar.gz \ - http://dl.openfoam.org/source/{OpenFOAMBase.version} && - wget -O ThirdParty-{OpenFOAMBase.v_major}-version-{OpenFOAMBase.v_major}.tar.gz \ - http://dl.openfoam.org/third-party/{OpenFOAMBase.v_major} - """, - ] + + @run_after("init") + def set_args(self): + """sets up download commands""" + executable_opts = [ + "-c", + f""" + wget -O OpenFOAM-{self.v_major}-{self.v_patch}.tar.gz \ + http://dl.openfoam.org/source/{self.version} && + wget -O ThirdParty-{self.v_major}-version-{self.v_major}.tar.gz \ + http://dl.openfoam.org/third-party/{self.v_major} + """, + ] + @sanity_function def validate_download(self): """Validate OpenFoam Downloaded""" return sn.all( [ - sn.path_isfile(f"OpenFOAM-{OpenFOAMBase.v_major}-{OpenFOAMBase.v_patch}.tar.gz"), - sn.path_isfile(f"ThirdParty-{OpenFOAMBase.v_major}-version-{OpenFOAMBase.v_major}.tar.gz"), + sn.path_isfile(f"OpenFOAM-{self.v_major}-{self.v_patch}.tar.gz"), + sn.path_isfile(f"ThirdParty-{self.v_major}-version-{self.v_major}.tar.gz"), ] ) diff --git a/tests/apps/openfoam/openfoam_org_dambreak.py b/tests/apps/openfoam/openfoam_org_dambreak.py index f0de2912..65a58f7c 100644 --- a/tests/apps/openfoam/openfoam_org_dambreak.py +++ b/tests/apps/openfoam/openfoam_org_dambreak.py @@ -15,13 +15,14 @@ class OpenFOAMDamBreakBase(OpenFOAMBase): num_tasks_per_node = 1 time_limit = "10m" valid_systems = ["archer2:compute"] - - - + @run_after("init") def setup_params(self): """sets up extra parameters""" + + + if self.current_system.name in ["archer2"]: freq = parameter(["2250000", "2000000"]) if self.current_system.name in ["archer2"]: @@ -53,13 +54,13 @@ def set_num_tasks(self): def setup_testcase(self): """set up test case""" - if (self.v_major == "10"): + if (self.version.startswith("10")): tutorial_sub_dir="tutorials/multiphase/interFoam/laminar/damBreak/damBreak" else: - if (self.v_major == "12"): + if (self.version.startswith("12")): tutorial_sub_dir="tutorials/incompressibleVoF/damBreak" else: - raise ValueError("Unsupported OpenFOAM version") + raise ValueError("Unsupported OpenFOAM version") self.prerun_cmds = [ @@ -88,8 +89,8 @@ class OpenFOAMDamBreakOneNode(OpenFOAMDamBreakBase): executable = "interFoam" executable_opts = ("").split() - - + valid_systems = ["archer2:compute","cirrus-ex:compute"] + num_tasks = 1 num_nodes = 1 @@ -170,6 +171,7 @@ class OpenFOAMDamBreakParallelModule(OpenFOAMDamBreakParallel): """OpenFOAM DamBreak test on 4 nodes with module""" valid_systems = ["archer2:compute","cirrus-ex:compute"] + executable = "interFoam" @run_before("run") diff --git a/tests/apps/openfoam/src/decomposeParDict b/tests/apps/openfoam/src/decomposeParDict new file mode 100644 index 00000000..18fa8a3d --- /dev/null +++ b/tests/apps/openfoam/src/decomposeParDict @@ -0,0 +1,42 @@ +/*--------------------------------*- C++ -*----------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | Website: https://openfoam.org + \\ / A nd | Version: 10 + \\/ M anipulation | +\*---------------------------------------------------------------------------*/ +FoamFile +{ + format ascii; + class dictionary; + location "system"; + object decomposeParDict; +} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +numberOfSubdomains 4; + +method simple; + +simpleCoeffs +{ + n (2 2 1); +} + +hierarchicalCoeffs +{ + n (1 1 1); + order xyz; +} + +manualCoeffs +{ + dataFile ""; +} + +distributed no; + +roots ( ); + + +// ************************************************************************* // \ No newline at end of file From fd5510dd6449bf0fa1125a854f26f797b59782de Mon Sep 17 00:00:00 2001 From: lparisi Date: Fri, 9 Jan 2026 16:26:16 +0000 Subject: [PATCH 06/18] Fetch only for one programming environment --- tests/apps/cp2k/cp2k_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/apps/cp2k/cp2k_check.py b/tests/apps/cp2k/cp2k_check.py index b9b84df5..f36b47a8 100644 --- a/tests/apps/cp2k/cp2k_check.py +++ b/tests/apps/cp2k/cp2k_check.py @@ -115,7 +115,8 @@ class FetchCP2K(rfm.RunOnlyRegressionTest): executable_opts = [f"https://github.com/cp2k/cp2k/archive/refs/tags/v{version}.tar.gz"] local = True valid_systems = ["cirrus-ex:login"] - valid_prog_environs = ["*"] + valid_prog_environs = ["PgEnv-gnu"] + @sanity_function def validate_download(self): From b2cc72cbf76b5bc94145d3fa75985cc5c899d540 Mon Sep 17 00:00:00 2001 From: lparisi Date: Fri, 9 Jan 2026 16:27:32 +0000 Subject: [PATCH 07/18] Restrict openfaom tests to module tests only --- tests/apps/openfoam/openfoam_org_dambreak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apps/openfoam/openfoam_org_dambreak.py b/tests/apps/openfoam/openfoam_org_dambreak.py index 65a58f7c..60363261 100644 --- a/tests/apps/openfoam/openfoam_org_dambreak.py +++ b/tests/apps/openfoam/openfoam_org_dambreak.py @@ -89,7 +89,7 @@ class OpenFOAMDamBreakOneNode(OpenFOAMDamBreakBase): executable = "interFoam" executable_opts = ("").split() - valid_systems = ["archer2:compute","cirrus-ex:compute"] + valid_systems = ["archer2:compute"] num_tasks = 1 num_nodes = 1 From 2f980e60845d939638955569515008c9d296ff4c Mon Sep 17 00:00:00 2001 From: lparisi Date: Fri, 9 Jan 2026 16:28:04 +0000 Subject: [PATCH 08/18] Updated reference to cirrus-ex for deepcam --- tests/libs/pytorch/deepcam.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/libs/pytorch/deepcam.py b/tests/libs/pytorch/deepcam.py index 8f077cf9..4b6f99e9 100644 --- a/tests/libs/pytorch/deepcam.py +++ b/tests/libs/pytorch/deepcam.py @@ -85,9 +85,10 @@ class DeepCAMCPUtest(DeepCAMBase): num_tasks = 32 time_limit = "1h" valid_systems = ["archer2:compute","cirrus-ex:compute"] + valid_prog_environs = ["PrgEnv-cray"] reference = {"archer2:compute": {"epoch-time": (272, -0.1, 0.1, "s")} - ,"cirrus-ex:compute": {"epoch-time": (119.7, -0.01, 0.01, "s")} + ,"cirrus-ex:compute": {"epoch-time": (40.08, -0.01, 0.01, "s")} } @run_after("init") From 85b1ab98e54bba434910b9e4eef480e088fdaa48 Mon Sep 17 00:00:00 2001 From: lucaparisi91 Date: Mon, 12 Jan 2026 14:14:11 +0000 Subject: [PATCH 09/18] Fixed broken openfoam test on Archer2, following work to adapt on cirrus --- tests/apps/openfoam/openfoam_org_base.py | 27 +++++--------- tests/apps/openfoam/openfoam_org_build.py | 8 ++--- tests/apps/openfoam/openfoam_org_dambreak.py | 37 +++++++++++++------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/apps/openfoam/openfoam_org_base.py b/tests/apps/openfoam/openfoam_org_base.py index e7b669f7..e3d605d3 100644 --- a/tests/apps/openfoam/openfoam_org_base.py +++ b/tests/apps/openfoam/openfoam_org_base.py @@ -14,20 +14,7 @@ class OpenFOAMBase(rfm.RunOnlyRegressionTest): use_multithreading = False tags = {"applications", "performance"} - @sanity_function - def assert_finished(self): - """Sanity check that simulation finished successfully""" - return sn.assert_found("End", self.stdout) - - @performance_function("seconds", perf_key="performance") - def extract_perf(self): - """Extract performance value to compare with reference value""" - return sn.extractsingle( - r"ExecutionTime\s+=\s+(?P