Rocm sn solver

.github/workflows/c-cpp.yml

@@ -8,32 +8,145 @@ on:
 
 jobs:
   unit_tests:
-      runs-on: ubuntu-latest
-      container: kitrt/test:latest
-      environment: coverage
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
-        OMP_NUM_THREADS: 1
-        OMP_DYNAMIC: FALSE
-        OPENBLAS_NUM_THREADS: 1
-        MKL_NUM_THREADS: 1
-
-      steps:
-      - uses: actions/checkout@v2
-
-      - name: Build code
-        run: |
-          git submodule update --init --recursive
-          mkdir build && cd build
-          cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DBUILD_ML=OFF -DBUILD_CODE_COV=ON ..
-          ninja
-
-      - name: Run unit tests
-        run: |
-          cd build
-          ./unit_tests 
-
-      - name: Code coverage
-        run: |
-          cpp-coveralls -r . -b "build/" -i "src/" -i "include/" --exclude "ext/" --gcov-options "\-lp" --verbose
+    runs-on: ubuntu-latest
+    container: kitrt/test:latest
+    defaults:
+      run:
+        working-directory: ${{ github.workspace }}
+    environment: coverage
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+      OMP_NUM_THREADS: 1
+      OMP_DYNAMIC: FALSE
+      OPENBLAS_NUM_THREADS: 1
+      MKL_NUM_THREADS: 1
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Build code
+      run: |
+        mkdir build && cd build
+        cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DBUILD_ML=OFF -DBUILD_CODE_COV=ON ..
+        ninja
+
+    - name: Run unit tests
+      run: |
+        cd build
+        ./unit_tests
+
+    - name: Code coverage
+      run: |
+        cpp-coveralls -r . -b "build/" -i "src/" -i "include/" --exclude "ext/" --gcov-options "\-lp" --verbose
+
+  rocm_build:
+    runs-on: ubuntu-latest
+    container: rocm/dev-ubuntu-22.04:7.2
+    env:
+      OMP_NUM_THREADS: 1
+      OMP_DYNAMIC: FALSE
+      OPENBLAS_NUM_THREADS: 1
+      MKL_NUM_THREADS: 1
+
+    steps:
+    - name: Install ROCm build dependencies
+      run: |
+        apt-get update
+        DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+          git \
+          libopenmpi-dev \
+          openmpi-bin \
+          libblas-dev \
+          liblapack-dev \
+          cmake \
+          ninja-build \
+          libvtk9-dev
+
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Build ROCm target
+      working-directory: ${{ github.workspace }}
+      run: |
+        ROCM_CLANGXX="/opt/rocm-7.2.0/lib/llvm/bin/clang++"
+        if [ ! -x "${ROCM_CLANGXX}" ] && [ -x "/opt/rocm/lib/llvm/bin/clang++" ]; then
+          ROCM_CLANGXX="/opt/rocm/lib/llvm/bin/clang++"
+        fi
+        if [ ! -x "${ROCM_CLANGXX}" ] && [ -x "/opt/rocm/llvm/bin/clang++" ]; then
+          ROCM_CLANGXX="/opt/rocm/llvm/bin/clang++"
+        fi
+        if [ ! -x "${ROCM_CLANGXX}" ]; then
+          echo "Could not find ROCm clang++ compiler." >&2
+          exit 1
+        fi
+        HIP_DIR=""
+        for CANDIDATE in \
+          "/opt/rocm-7.2.0/lib/cmake/hip" \
+          "/opt/rocm/lib/cmake/hip" \
+          "/opt/rocm/cmake/hip"
+        do
+          if [ -f "${CANDIDATE}/hip-config.cmake" ] || [ -f "${CANDIDATE}/hipConfig.cmake" ]; then
+            HIP_DIR="${CANDIDATE}"
+            break
+          fi
+        done
+        if [ -z "${HIP_DIR}" ]; then
+          echo "Could not find hip package config directory." >&2
+          exit 1
+        fi
+        cmake -G Ninja -S . -B build_rocm \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_CXX_COMPILER="${ROCM_CLANGXX}" \
+          -DCMAKE_HIP_COMPILER="${ROCM_CLANGXX}" \
+          -Dhip_DIR="${HIP_DIR}" \
+          -DBUILD_MPI=ON \
+          -DBUILD_CUDA_HPC=OFF \
+          -DBUILD_HIP_HPC=ON \
+          -DBUILD_ML=OFF
+        cmake --build build_rocm -j2
+
+  cuda_build:
+    runs-on: ubuntu-latest
+    container: nvidia/cuda:12.4.1-devel-ubuntu22.04
+    env:
+      OMP_NUM_THREADS: 1
+      OMP_DYNAMIC: FALSE
+      OPENBLAS_NUM_THREADS: 1
+      MKL_NUM_THREADS: 1
+
+    steps:
+    - name: Install CUDA build dependencies
+      run: |
+        apt-get update
+        DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+          git \
+          libopenmpi-dev \
+          openmpi-bin \
+          libblas-dev \
+          liblapack-dev \
+          cmake \
+          ninja-build \
+          libvtk9-dev
+
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Build CUDA target and tests
+      working-directory: ${{ github.workspace }}
+      run: |
+        cmake -G Ninja -S . -B build_cuda \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DBUILD_TESTING=ON \
+          -DBUILD_MPI=ON \
+          -DBUILD_CUDA_HPC=ON \
+          -DBUILD_HIP_HPC=OFF \
+          -DBUILD_ML=OFF \
+          -DCMAKE_CUDA_ARCHITECTURES=70
+        cmake --build build_cuda -j2
+
+

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required( VERSION 3.16 )
-project( KiT-RT VERSION 0.1.0 LANGUAGES CXX )
+project( KiT-RT VERSION 1.2.0 LANGUAGES CXX )
 
 ### OPTIONS #####################################
 option( BUILD_TESTING "builds all available unit tests" OFF )
@@ -10,12 +10,16 @@ option( BUILD_CODE_COV "enables compiler option required for code coverage analy
 option( BUILD_ML "enables build with tensorflow backend access" OFF )
 option( BUILD_MPI "enables build with MPI access" OFF )
 option( BUILD_CUDA_HPC "enables CUDA backend for SN HPC solver (MPI rank to GPU mapping)" OFF )
+option( BUILD_HIP_HPC "enables HIP/ROCm backend for SN HPC solver (MPI rank to GPU mapping)" OFF )
 #################################################
 
 
 ### COMPILER ####################################
 set( CMAKE_CXX_STANDARD 17 )
 set( CMAKE_CXX_STANDARD_REQUIRED ON )
+if( BUILD_CUDA_HPC AND BUILD_HIP_HPC )
+    message( FATAL_ERROR "BUILD_CUDA_HPC and BUILD_HIP_HPC cannot both be enabled." )
+endif()
 if( BUILD_CUDA_HPC )
     enable_language( CUDA )
     if( CMAKE_VERSION VERSION_LESS 3.18 )
@@ -27,9 +31,25 @@ if( BUILD_CUDA_HPC )
     set( CMAKE_CUDA_STANDARD_REQUIRED ON )
     set( CMAKE_CUDA_EXTENSIONS OFF )
 endif()
+if( BUILD_HIP_HPC )
+    if( CMAKE_VERSION VERSION_LESS 3.21 )
+        message( FATAL_ERROR "BUILD_HIP_HPC requires CMake >= 3.21 for HIP language support." )
+    endif()
+    enable_language( HIP )
+    set( CMAKE_HIP_STANDARD 17 )
+    set( CMAKE_HIP_STANDARD_REQUIRED ON )
+    set( CMAKE_HIP_EXTENSIONS OFF )
+endif()
 set( KITRT_RELEASE_OPTIONS -march=native -w )
 set( KITRT_RELWITHDEBINFO_OPTIONS -march=native -pg -no-pie )
 set( KITRT_DEBUG_OPTIONS -Wall -Wextra -Wpedantic )
+if( BUILD_HIP_HPC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|X86_64|amd64|AMD64" )
+    # ROCm clang can expose AVX-512 feature macros that trigger an incompatible
+    # Blaze SIMD floor implementation path on some hosts.
+    list( APPEND KITRT_RELEASE_OPTIONS -mno-avx512f -mno-avx512bw -mno-avx512dq )
+    list( APPEND KITRT_RELWITHDEBINFO_OPTIONS -mno-avx512f -mno-avx512bw -mno-avx512dq )
+    message( STATUS "HIP host C++ compile options: disabling AVX-512 for Blaze compatibility" )
+endif()
 if( BUILD_UNITY AND NOT BUILD_CODE_COV )
     message( STATUS "Unity build enabled" )
     set( CMAKE_UNITY_BUILD ON )
@@ -57,6 +77,14 @@ else()
     message( STATUS "CUDA HPC solver: disabled" )
 endif()
 
+if( BUILD_HIP_HPC )
+    add_compile_definitions( KITRT_ENABLE_HIP_HPC )
+    find_package( hip REQUIRED CONFIG )
+    message( STATUS "HIP HPC solver: enabled" )
+else()
+    message( STATUS "HIP HPC solver: disabled" )
+endif()
+
 message(STATUS "MPI build flag: ${BUILD_MPI}")
 if( BUILD_MPI )
     add_definitions(-DIMPORT_MPI)
@@ -113,6 +141,16 @@ if( BUILD_CUDA_HPC )
     endif()
 endif()
 
+if( BUILD_HIP_HPC )
+    if( TARGET hip::host )
+        # Link only the host runtime to avoid propagating HIP device compile flags
+        # (e.g. --offload-arch=...) into non-HIP C++ translation units.
+        list( APPEND CORE_LIBRARIES hip::host )
+    else()
+        message( FATAL_ERROR "Could not find HIP runtime target (hip::host)." )
+    endif()
+endif()
+
 
 #################################################
 
@@ -136,6 +174,10 @@ if( BUILD_CUDA_HPC )
     # imported target features inject an older fallback standard.
     set_source_files_properties( src/solvers/snsolver_hpc.cu PROPERTIES COMPILE_FLAGS "--std=c++17" )
 endif()
+if( BUILD_HIP_HPC )
+    list( APPEND SRCS "src/solvers/snsolver_hpc.hip" "include/solvers/snsolver_hpc_hip.hpp" )
+    set_source_files_properties( src/solvers/snsolver_hpc.hip PROPERTIES LANGUAGE HIP )
+endif()
 set( EXCLUDE_DIR "/gui/" )
 foreach( TMP_PATH ${SRCS} )
     string( FIND ${TMP_PATH} ${EXCLUDE_DIR} EXCLUDE_DIR_FOUND )
@@ -163,6 +205,12 @@ if( BUILD_CUDA_HPC )
         PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON CUDA_EXTENSIONS OFF
     )
 endif()
+if( BUILD_HIP_HPC )
+    set_target_properties(
+        ${CMAKE_PROJECT_NAME}
+        PROPERTIES HIP_STANDARD 17 HIP_STANDARD_REQUIRED ON HIP_EXTENSIONS OFF
+    )
+endif()
 target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
 target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELWITHDEBINFO>>:${KITRT_RELWITHDEBINFO_OPTIONS}>" )
 target_compile_options( ${CMAKE_PROJECT_NAME} PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:RELEASE>>:${KITRT_RELEASE_OPTIONS}>" )
@@ -204,6 +252,12 @@ if( BUILD_TESTING )
             PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON CUDA_EXTENSIONS OFF
         )
     endif()
+    if( BUILD_HIP_HPC )
+        set_target_properties(
+            unit_tests
+            PROPERTIES HIP_STANDARD 17 HIP_STANDARD_REQUIRED ON HIP_EXTENSIONS OFF
+        )
+    endif()
     target_compile_options( unit_tests PUBLIC "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${KITRT_DEBUG_OPTIONS}>" )
     if( BUILD_CODE_COV )
         if( CMAKE_COMPILER_IS_GNUCXX )

README.md

@@ -143,9 +143,9 @@ singularity exec tools/singularity/kit_rt_MPI.sif \
   mpirun -np 4 ./build_singularity_mpi/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cpu_order2.cfg
 ```
 
-### 3. CPU + CUDA (single or multi-GPU via MPI)
+### 3. CPU + GPU backend (single or multi-GPU via MPI)
 
-#### 3a) Singularity installation
+#### 3a) CUDA (Singularity installation)
 ```bash
 cd tools/singularity
 sudo singularity build kit_rt_MPI_cuda.sif kit_rt_MPI_cuda.def
@@ -162,6 +162,23 @@ singularity exec --nv tools/singularity/kit_rt_MPI_cuda.sif \
 
 When compiled with `-DBUILD_CUDA_HPC=ON`, HPC runs use the CUDA backend if a GPU is visible, and fall back to CPU if no GPU is detected.
 
+#### 3b) ROCm/HIP (Singularity installation)
+```bash
+cd tools/singularity
+sudo singularity build kit_rt_MPI_rocm72.sif kit_rt_MPI_rocm72.def
+cd ../..
+mkdir -p build_singularity_rocm72
+cd build_singularity_rocm72
+singularity exec --rocm ../tools/singularity/kit_rt_MPI_rocm72.sif \
+  cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_MPI=ON -DBUILD_CUDA_HPC=OFF -DBUILD_HIP_HPC=ON -DBUILD_ML=OFF ..
+singularity exec --rocm ../tools/singularity/kit_rt_MPI_rocm72.sif make -j
+cd ..
+singularity exec --rocm tools/singularity/kit_rt_MPI_rocm72.sif \
+  mpirun -np 2 ./build_singularity_rocm72/KiT-RT tests/input/validation_tests/SN_solver_hpc/lattice_hpc_200_cuda_order2.cfg
+```
+
+When compiled with `-DBUILD_HIP_HPC=ON`, HPC runs use the HIP backend if a ROCm-capable GPU is visible, and fall back to CPU if no GPU is detected.
+
 ### 4. Build with TensorFlow backend (CPU + OpenMP only)
 
 ```bash