Parallel I/O integration

CMakeLists.txt

@@ -7,14 +7,43 @@ project(
 )
 
 option(CABLE_MPI "Build the MPI executable" OFF)
+option(CABLE_TESTS "Build CABLE tests" OFF)
 
 # third party libs
 if(CABLE_MPI)
     find_package(MPI REQUIRED COMPONENTS Fortran)
+    find_package(PIO COMPONENTS Fortran QUIET)
+    if(TARGET PIO::PIO_Fortran)
+        message(STATUS "Found PIO_Fortran: ${PIO_DIR}")
+    endif()
 endif()
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(NETCDF REQUIRED IMPORTED_TARGET "netcdf-fortran")
 
+if(CABLE_TESTS)
+    enable_testing()
+    include(FetchContent)
+    if(CABLE_MPI)
+        option(FORTUNO_WITH_MPI "Fortuno: whether to build the MPI interface" ON)
+        FetchContent_Declare(
+            FortunoMPI
+            GIT_REPOSITORY https://github.com/fortuno-repos/fortuno
+            GIT_TAG main
+        )
+        FetchContent_MakeAvailable(FortunoMPI)
+        set(fortuno_libs Fortuno::fortuno_mpi)
+    else()
+        option(FORTUNO_WITH_MPI "Fortuno: whether to build the MPI interface" OFF)
+        FetchContent_Declare(Fortuno
+            GIT_REPOSITORY https://github.com/fortuno-repos/fortuno
+            GIT_TAG main
+            FIND_PACKAGE_ARGS CONFIG
+        )
+        FetchContent_MakeAvailable(Fortuno)
+        set(fortuno_libs Fortuno::fortuno_serial)
+    endif ()
+endif()
+
 set(CABLE_Intel_Fortran_FLAGS -fp-model precise)
 set(CABLE_Intel_Fortran_FLAGS_DEBUG -O0 -g -traceback -fpe0)
 set(CABLE_Intel_Fortran_FLAGS_RELEASE -O2)
@@ -239,6 +268,7 @@ else()
         src/util/cable_common.F90
         src/shared/casa_offline_inout.F90
         src/shared/casa_ncdf.F90
+        src/offline/cable_io_decomp.F90
         src/offline/cable_iovars.F90
         src/offline/cable_surface_types.F90
         src/offline/cable_define_types.F90
@@ -271,13 +301,26 @@ else()
         src/offline/spincasacnp.F90
         src/util/cable_climate_type_mod.F90
         src/util/masks_cbl.F90
+        src/util/cable_array_utils.F90
+        src/util/netcdf/cable_netcdf_decomp_util.F90
+        src/util/netcdf/cable_netcdf.F90
+        src/util/netcdf/cable_netcdf_internal.F90
+        src/util/netcdf/cable_netcdf_stub_types.F90
+        src/util/netcdf/nf90/cable_netcdf_nf90.F90
     )
 
     target_link_libraries(cable_common PRIVATE PkgConfig::NETCDF)
 
     if(CABLE_MPI)
-    target_compile_definitions(cable_common PRIVATE __MPI__)
-    target_link_libraries(cable_common PRIVATE MPI::MPI_Fortran)
+        target_compile_definitions(cable_common PRIVATE __MPI__)
+        target_link_libraries(cable_common PRIVATE MPI::MPI_Fortran)
+    endif()
+
+    if(TARGET PIO::PIO_Fortran)
+        target_link_libraries(cable_common PRIVATE PIO::PIO_Fortran)
+        target_sources(cable_common PRIVATE src/util/netcdf/pio/cable_netcdf_pio.F90)
+    else()
+        target_sources(cable_common PRIVATE src/util/netcdf/pio/cable_netcdf_pio_stub.F90)
     endif()
 
     if(CABLE_MPI)
@@ -302,4 +345,33 @@ else()
         install(TARGETS cable RUNTIME)
     endif()
 
+    if (CABLE_TESTS)
+        add_executable(
+            cable-tests
+            tests/cable_tests.F90
+            tests/fixtures.F90
+            tests/utils/file_utils.F90
+            tests/test_cable_netcdf.F90
+        )
+        if(CABLE_MPI)
+            target_sources(cable-tests PRIVATE tests/fortuno_interface_mpi.f90)
+        else()
+            target_sources(cable-tests PRIVATE tests/fortuno_interface_serial.f90)
+        endif()
+        target_link_libraries(cable-tests PRIVATE cable_common ${fortuno_libs})
+        if(CABLE_MPI)
+            add_test(NAME cable-tests-serial
+                COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 $<TARGET_FILE:cable-tests> ~parallel
+            )
+            set_tests_properties(cable-tests-serial PROPERTIES PROCESSORS 1)
+            add_test(NAME cable-tests-parallel
+                COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 $<TARGET_FILE:cable-tests> parallel
+            )
+            set_tests_properties(cable-tests-parallel PROPERTIES PROCESSORS 4)
+        else()
+            add_test(NAME cable-tests-serial
+                COMMAND $<TARGET_FILE:cable-tests> ~parallel
+            )
+        endif()
+    endif()
 endif()

build.bash

@@ -17,6 +17,9 @@ options below will be passed to CMake when generating the build system.
 Options:
   -c, --clean   Delete build directory before invoking CMake.
   -m, --mpi     Compile MPI executable.
+  -p, --parallelio
+                Enable parallel I/O support. This flag requires that --mpi is
+                also set.
   -C, --compiler <compiler>
                 Specify the compiler to use.
   -n, --ncpus <ncpus>
@@ -63,6 +66,9 @@ while [ ${#} -gt 0 ]; do
             mpi=1
             cmake_args+=(-DCABLE_MPI="ON")
             ;;
+        -p|--parallelio)
+            pio=1
+            ;;
         -l|--library)
             build_args+=(--target cable_science)
             cmake_args+=(-DCABLE_LIBRARY="ON")
@@ -98,9 +104,15 @@ if hostname -f | grep gadi.nci.org.au > /dev/null; then
     module add netcdf/4.6.3
     case ${compiler} in
         intel)
-            module add intel-compiler/2019.5.281
+            module add intel-compiler-llvm/2025.0.4
             compiler_lib_install_dir=Intel
-            [[ -n ${mpi} ]] && module add intel-mpi/2019.5.281
+            [[ -n ${mpi} ]] && module add openmpi/4.1.7
+            # This is required so that the Parallel IO library is discoverable
+            # via CMake's `find_package` mechanism:
+            # TODO(Sean): This install of Parallel IO is specific to
+            # openmpi/4.1.7. We need a better way to provide this library on
+            # Gadi.
+            [[ -n ${pio} ]] && prepend_path CMAKE_PREFIX_PATH "/g/data/tm70/sb8430/parallelio_install"
             ;;
         gnu)
             module add gcc/13.2.0
@@ -123,6 +135,15 @@ if hostname -f | grep gadi.nci.org.au > /dev/null; then
         prepend_path CMAKE_PREFIX_PATH "${OPENMPI_BASE}/include/${compiler_lib_install_dir}"
     fi
 
+    if [[ -n ${pio} ]]; then
+        # The NetCDF Fortran version must be consistent with the version used in Parallel IO
+        # TODO(Sean): we need a better way to provide these libraries on Gadi
+        prepend_path CMAKE_PREFIX_PATH "/g/data/tm70/sb8430/spack/0.22/release/linux-rocky8-x86_64_v4/intel-2021.10.0/netcdf-c-4.9.2-oxepdmgcx6raxo4vi4teu45qqr63v3uj"
+        prepend_path PKG_CONFIG_PATH "/g/data/tm70/sb8430/spack/0.22/release/linux-rocky8-x86_64_v4/intel-2021.10.0/netcdf-c-4.9.2-oxepdmgcx6raxo4vi4teu45qqr63v3uj/lib/pkgconfig"
+        prepend_path CMAKE_PREFIX_PATH "/g/data/tm70/sb8430/spack/0.22/release/linux-rocky8-x86_64_v4/intel-2021.10.0/netcdf-fortran-4.6.1-eq777uogbelnhv43ln6jyub2gbmos42x"
+        prepend_path PKG_CONFIG_PATH "/g/data/tm70/sb8430/spack/0.22/release/linux-rocky8-x86_64_v4/intel-2021.10.0/netcdf-fortran-4.6.1-eq777uogbelnhv43ln6jyub2gbmos42x/lib/pkgconfig"
+    fi
+
 elif hostname -f | grep -E '(mc16|mcmini)' > /dev/null; then
     : "${compiler:=gnu}"
 

src/offline/cable_abort.F90

@@ -20,55 +20,37 @@
 
 MODULE cable_abort_module
 
-  USE cable_IO_vars_module, ONLY: check, logn
-  IMPLICIT NONE
-
-CONTAINS
+  USE iso_fortran_env, ONLY: error_unit
+  USE cable_mpi_mod, ONLY: mpi_grp_t
 
-  !==============================================================================
-  !
-  ! Name: abort
-  !
-  ! Purpose: Prints an error message and stops the code
-  !
-  ! CALLed from: get_default_inits
-  !              get_restart_data
-  !              get_default_lai
-  !              open_met_file
-  !              get_met_data
-  !              load_parameters
-  !              open_output_file
-  !              write_output
-  !              read_gridinfo
-  !              countpatch
-  !              get_type_parameters
-  !              readpar_i
-  !              readpar_r
-  !              readpar_rd
-  !              readpar_r2
-  !              readpar_r2d
-  !              define_output_variable_r1
-  !              define_output_variable_r2
-  !              define_output_parameter_r1
-  !              define_output_parameter_r2
-  !              write_output_variable_r1
-  !              write_output_variable_r2
-  !              write_output_parameter_r1
-  !              write_output_parameter_r1d
-  !              write_output_parameter_r2
-  !              write_output_parameter_r2d
-  !
-  !==============================================================================
+  IMPLICIT NONE
 
-  SUBROUTINE abort(message)
+  TYPE(mpi_grp_t), PRIVATE :: mpi_grp_global
 
-    ! Input arguments
-    CHARACTER(LEN=*), INTENT(IN) :: message
+CONTAINS
 
-    WRITE (*, *) message
-    STOP 1
+  SUBROUTINE cable_abort_module_init(mpi_grp)
+    !! Initialise abort module
+    TYPE(mpi_grp_t), intent(in) :: mpi_grp
+    mpi_grp_global = mpi_grp
+  END SUBROUTINE
+
+  SUBROUTINE cable_abort(message, file, line)
+    !! Print the error message and stop the code
+    CHARACTER(LEN=*), INTENT(IN) :: message !! Error message
+    CHARACTER(LEN=*), INTENT(IN), OPTIONAL :: file
+    INTEGER, INTENT(IN), OPTIONAL :: line
+    CHARACTER(5) :: line_string
+
+    IF (present(file) .AND. present(line)) THEN
+      WRITE (line_string, "(I5)") line
+      WRITE (error_unit, *) file // ":" // trim(adjustl(line_string)) // ": " // message
+    ELSE
+      WRITE (error_unit, *) message
+    END IF
+    call mpi_grp_global%abort()
 
-  END SUBROUTINE abort
+  END SUBROUTINE
 
   !==============================================================================
   !
@@ -123,72 +105,4 @@ SUBROUTINE nc_abort(ok, message)
 
   END SUBROUTINE nc_abort
 
-  !==============================================================================
-  !
-  ! Name: range_abort
-  !
-  ! Purpose: Prints an error message and localisation information then stops the
-  !          code
-  !
-  ! CALLed from: write_output_variable_r1
-  !              write_output_variable_r2
-  !
-  ! MODULEs used: cable_def_types_mod
-  !               cable_IO_vars_module
-  !
-  !==============================================================================
-
-  SUBROUTINE range_abort(vname, ktau, met, value, var_range, i, xx, yy)
-
-    USE cable_def_types_mod, ONLY: met_type
-    USE cable_IO_vars_module, ONLY: latitude, longitude, &
-                                    landpt, lat_all, lon_all
-
-    ! Input arguments
-    CHARACTER(LEN=*), INTENT(IN) :: vname
-
-    INTEGER, INTENT(IN) :: &
-      ktau, & ! time step
-      i       ! tile number along mp
-
-    REAL, INTENT(IN) :: &
-      xx, & ! coordinates of erroneous grid square
-      yy    ! coordinates of erroneous grid square
-
-    TYPE(met_type), INTENT(IN) :: met  ! met data
-
-    REAL(4), INTENT(IN) :: value ! value deemed to be out of range
-
-    REAL, INTENT(IN) :: var_range(2) ! appropriate var range
-
-    INTEGER :: iunit
-
-    IF (check%exit) THEN
-      iunit = 6
-    ELSE
-      iunit = logn ! warning
-    END IF
-
-    WRITE (iunit, *) "in SUBR range_abort: Out of range"
-    WRITE (iunit, *) "for var ", vname ! error from subroutine
-
-    ! patch(i)%latitude, patch(i)%longitude
-    WRITE (iunit, *) 'Site lat, lon:', xx, yy
-    WRITE (iunit, *) 'Output timestep', ktau, &
-      ', or ', met%hod(i), ' hod, ', &
-      INT(met%doy(i)), 'doy, ', &
-      INT(met%year(i))
-
-    WRITE (iunit, *) 'Specified acceptable range (cable_checks.f90):', &
-      var_range(1), 'to', var_range(2)
-
-    WRITE (iunit, *) 'Value:', value
-
-    IF (check%exit) THEN
-      STOP
-    END IF
-
-  END SUBROUTINE range_abort
-
-  !==============================================================================
 END MODULE cable_abort_module

src/offline/cable_checks.F90

@@ -30,8 +30,9 @@ MODULE cable_checks_module
   ! particular sections of the code - largely for diagnostics/fault finding.
   ! rh_sh - converts relative to sensible humidity if met file units require it
   !
-  USE cable_IO_vars_module, ONLY: patch
-  USE cable_abort_module, ONLY: range_abort
+  USE iso_fortran_env, ONLY: error_unit
+  USE cable_IO_vars_module, ONLY: patch, check, logn
+  USE cable_abort_module, ONLY: cable_abort
   USE cable_def_types_mod
   USE cable_common_module, ONLY: cable_user
 
@@ -212,6 +213,52 @@ MODULE cable_checks_module
 
 CONTAINS
 
+  SUBROUTINE range_abort(vname, ktau, met, value, var_range, i, xx, yy)
+    !! Prints an error message and localisation information then stops the code
+
+    CHARACTER(LEN=*), INTENT(IN) :: vname
+
+    INTEGER, INTENT(IN) :: &
+      ktau, & ! time step
+      i       ! tile number along mp
+
+    REAL, INTENT(IN) :: &
+      xx, & ! coordinates of erroneous grid square
+      yy    ! coordinates of erroneous grid square
+
+    TYPE(met_type), INTENT(IN) :: met  ! met data
+
+    REAL(4), INTENT(IN) :: value ! value deemed to be out of range
+
+    REAL, INTENT(IN) :: var_range(2) ! appropriate var range
+
+    INTEGER :: iunit
+
+    IF (check%exit) THEN
+      iunit = error_unit
+    ELSE
+      iunit = logn ! warning
+    END IF
+
+    WRITE (iunit, *) "in SUBR range_abort: Out of range"
+    WRITE (iunit, *) "for var ", vname ! error from subroutine
+
+    ! patch(i)%latitude, patch(i)%longitude
+    WRITE (iunit, *) 'Site lat, lon:', xx, yy
+    WRITE (iunit, *) 'Output timestep', ktau, &
+      ', or ', met%hod(i), ' hod, ', &
+      INT(met%doy(i)), 'doy, ', &
+      INT(met%year(i))
+
+    WRITE (iunit, *) 'Specified acceptable range (cable_checks.f90):', &
+      var_range(1), 'to', var_range(2)
+
+    WRITE (iunit, *) 'Value:', value
+
+    IF (check%exit) CALL cable_abort("Aborting...")
+
+  END SUBROUTINE range_abort
+
   SUBROUTINE check_range_d1(vname, parameter_r1, parameter_range, ktau, met)
 
     CHARACTER(LEN=*) :: vname