AMDFlang compiler for MFC

.github/workflows/bench.yml

@@ -70,6 +70,14 @@ jobs:
             device: gpu
             interface: omp
             build_script: "bash .github/workflows/frontier/build.sh gpu omp bench"
+          - cluster: frontier
+            name: Oak Ridge | Frontier (AMD)
+            group: phoenix
+            labels: frontier_amd
+            flag: famd
+            device: gpu
+            interface: omp
+            build_script: "bash .github/workflows/frontier_amd/build.sh gpu omp bench"
     runs-on:
       group: ${{ matrix.group }}
       labels: ${{ matrix.labels }}

.github/workflows/frontier_amd/bench.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+n_ranks=12
+device_opts=""
+if [ "$job_device" = "gpu" ]; then
+    gpus=$(rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' ')
+    n_ranks=$(echo "$gpus" | wc -w)         # number of GPUs on node
+    gpu_ids=$(echo "$gpus" | tr ' ' '\n' | tr '\n' ' ' | sed 's/ $//')  # GPU IDs from rocm-smi
+    device_opts+="--gpu"
+    if [ "$job_interface" = "acc" ]; then
+        device_opts+=" acc"
+    elif [ "$job_interface" = "omp" ]; then
+        device_opts+=" mp"
+    fi
+    device_opts+=" -g $gpu_ids"
+fi
+
+if [ "$job_device" = "gpu" ]; then
+    ./mfc.sh bench --mem 12 -j $n_ranks -o "$job_slug.yaml" -- -c frontier_amd $device_opts -n $n_ranks
+else
+    ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c frontier_amd $device_opts -n $n_ranks
+fi

.github/workflows/frontier_amd/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+job_device=$1
+job_interface=$2
+run_bench=$3
+build_opts=""
+if [ "$job_device" = "gpu" ]; then
+  build_opts+="--gpu"
+  if [ "$job_interface" = "acc" ]; then
+      build_opts+=" acc"
+  elif [ "$job_interface" = "omp" ]; then
+      build_opts+=" mp"
+  fi
+fi
+
+. ./mfc.sh load -c famd -m g
+
+if [ "$run_bench" == "bench" ]; then
+    for dir in benchmarks/*/; do
+        dirname=$(basename "$dir")
+        ./mfc.sh run "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts
+    done
+else
+    ./mfc.sh test -a --dry-run -j 8 $build_opts
+fi
+

.github/workflows/frontier_amd/submit-bench.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+    echo "Usage: $0 [script.sh] [cpu|gpu]"
+}
+
+if [ ! -z "$1" ]; then
+    sbatch_script_contents=`cat $1`
+else
+    usage
+    exit 1
+fi
+
+if [ "$2" = "cpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 32                       # Number of cores required"
+elif [ "$2" = "gpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 8                       # Number of cores required"
+else
+    usage; exit 1
+fi
+
+
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
+
+sbatch <<EOT
+#!/bin/bash
+#SBATCH -JMFC-$job_slug            # Job name
+#SBATCH -A CFD154                  # charge account
+#SBATCH -N 1                       # Number of nodes required
+$sbatch_device_opts
+#SBATCH -t 02:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -p extended                # Extended partition for shorter queues
+
+set -e
+set -x
+
+cd "\$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
+
+job_slug="$job_slug"
+job_device="$2"
+job_interface="$3"
+
+. ./mfc.sh load -c famd -m g
+
+$sbatch_script_contents
+
+EOT
+

.github/workflows/frontier_amd/submit.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+    echo "Usage: $0 [script.sh] [cpu|gpu]"
+}
+
+if [ ! -z "$1" ]; then
+    sbatch_script_contents=`cat $1`
+else
+    usage
+    exit 1
+fi
+
+if [ "$2" = "cpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 32                       # Number of cores required"
+elif [ "$2" = "gpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 8                       # Number of cores required"
+else
+    usage
+    exit 1
+fi
+
+
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
+
+sbatch <<EOT
+#!/bin/bash
+#SBATCH -J MFC-$job_slug            # Job name
+#SBATCH -A CFD154                  # charge account
+#SBATCH -N 1                       # Number of nodes required
+$sbatch_device_opts
+#SBATCH -t 05:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -p extended                # Extended partition for shorter queues
+#SBATCH -W                         # Do not exit until the submitted job terminates.
+
+set -e
+set -x
+
+cd "\$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
+
+job_slug="$job_slug"
+job_device="$2"
+job_interface="$3"
+
+. ./mfc.sh load -c famd -m $([ "$2" = "gpu" ] && echo "g" || echo "c")
+
+$sbatch_script_contents
+
+EOT
+

.github/workflows/frontier_amd/test.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+gpus=`rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' '`
+ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
+
+device_opts=""
+if [ "$job_device" = "gpu" ]; then
+    device_opts+="--gpu"
+    if [ "$job_interface" = "acc" ]; then
+        device_opts+=" acc"
+    elif [ "$job_interface" = "omp" ]; then
+        device_opts+=" mp"
+    fi
+fi
+
+if [ "$job_device" = "gpu" ]; then
+    ./mfc.sh test -a  --max-attempts 3 -j $ngpus $device_opts -- -c frontier_amd
+else
+    ./mfc.sh test -a --max-attempts 3 -j 32 --no-gpu -- -c frontier_amd
+fi

.github/workflows/test.yml

@@ -127,6 +127,10 @@ jobs:
             cluster_name: 'Oak Ridge | Frontier'
             device: 'cpu'
             interface: 'none'
+          - lbl: 'frontier_amd'
+            cluster_name: 'Oak Ridge | Frontier (AMD)'
+            device: 'gpu'
+            interface: 'omp'
     runs-on:
       group:  phoenix
       labels: ${{ matrix.lbl }}
@@ -150,6 +154,14 @@ jobs:
         if:   matrix.lbl == 'frontier'
         run:  bash .github/workflows/frontier/submit.sh .github/workflows/frontier/test.sh ${{matrix.device}} ${{ matrix.interface }}
 
+      - name: Build
+        if:   matrix.lbl == 'frontier_amd'
+        run:  bash .github/workflows/frontier_amd/build.sh ${{ matrix.device }} ${{ matrix.interface }}
+
+      - name: Test
+        if:   matrix.lbl == 'frontier_amd'
+        run:  bash .github/workflows/frontier_amd/submit.sh .github/workflows/frontier_amd/test.sh ${{matrix.device}} ${{ matrix.interface }}  
+
       - name: Print Logs
         if:   always()
         run:  cat test-${{ matrix.device }}-${{ matrix.interface }}.out
@@ -160,3 +172,10 @@ jobs:
         with:
           name: logs-${{ strategy.job-index }}-${{ matrix.device }}-${{ matrix.interface }}
           path: test-${{ matrix.device }}-${{ matrix.interface }}.out
+
+      - name: Archive Logs
+        uses: actions/upload-artifact@v4
+        if:   matrix.lbl == 'frontier_amd'
+        with:
+          name: logs-${{ strategy.job-index }}-${{ matrix.device }}-${{ matrix.interface }}
+          path: test-${{ matrix.device }}-${{ matrix.interface }}.out

CMakeLists.txt

@@ -310,7 +310,18 @@ endif()
 # and generate documentation. Instead, we can simply include the list of .f90
 # files that will eventually be used to compile <target>.
 
-macro(HANDLE_SOURCES target useCommon)
+macro(HANDLE_SOURCES target useCommon useOpenACC useOpenMP)
+
+    if (${useOpenACC} AND ${useOpenMP})
+      message(FATAL_ERROR "OpenACC and OpenMP at same time is unsupported.")
+    elseif (${useOpenACC})
+      set(MFC_GPU_MODE "OpenACC")
+    elseif (${useOpenMP})
+      set(MFC_GPU_MODE "OpenMP")
+    else()
+      set(MFC_GPU_MODE "")
+    endif()
+
     set(${target}_DIR "${CMAKE_SOURCE_DIR}/src/${target}")
     set(common_DIR    "${CMAKE_SOURCE_DIR}/src/common")
 
@@ -371,7 +382,8 @@ macro(HANDLE_SOURCES target useCommon)
                                  -D MFC_${CMAKE_Fortran_COMPILER_ID}
                                  -D MFC_${${target}_UPPER}
                                  -D MFC_COMPILER="${CMAKE_Fortran_COMPILER_ID}"
-                                 -D MFC_CASE_OPTIMIZATION=False
+                                 -D MFC_GPU_MODE="${MFC_GPU_MODE}"
+				                 -D MFC_CASE_OPTIMIZATION=False
                                  -D chemistry=False
                                  --line-numbering
                                  --no-folding
@@ -388,11 +400,10 @@ macro(HANDLE_SOURCES target useCommon)
 endmacro()
 
 
-HANDLE_SOURCES(pre_process ON)
-HANDLE_SOURCES(simulation ON)
-HANDLE_SOURCES(post_process ON)
-HANDLE_SOURCES(syscheck OFF)
-
+HANDLE_SOURCES(pre_process ON OFF OFF)
+HANDLE_SOURCES(simulation ON ${MFC_OpenACC} ${MFC_OpenMP})
+HANDLE_SOURCES(post_process ON OFF OFF)
+HANDLE_SOURCES(syscheck OFF ${MFC_OpenACC} ${MFC_OpenMP})
 
 # MFC_SETUP_TARGET: Given a target (herein <target>), this macro creates a new
 # executable <target> with the appropriate sources, compiler definitions, and
@@ -430,6 +441,7 @@ function(MFC_SETUP_TARGET)
 
     foreach (a_target ${IPO_TARGETS})
         set_target_properties(${a_target} PROPERTIES Fortran_PREPROCESS ON)
+	    message(STATUS ${CMAKE_Fortran_COMPILER_ID})
 
         target_include_directories(${a_target} PRIVATE
             "${CMAKE_SOURCE_DIR}/src/common"
@@ -449,9 +461,14 @@ function(MFC_SETUP_TARGET)
 
         if (MFC_MPI AND ARGS_MPI)
             find_package(MPI COMPONENTS Fortran REQUIRED)
-
+	    
             target_compile_definitions(${a_target} PRIVATE MFC_MPI)
-            target_link_libraries     (${a_target} PRIVATE MPI::MPI_Fortran)
+            if(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+                target_compile_options(${a_target} PRIVATE "$ENV{CRAY_MPICH_INC}")
+                target_link_libraries(${a_target} PRIVATE $ENV{CRAY_MPICH_LIB})
+            else() 
+                target_link_libraries(${a_target} PRIVATE MPI::MPI_Fortran)
+            endif()
         endif()
 
         if (ARGS_SILO)
@@ -469,9 +486,11 @@ function(MFC_SETUP_TARGET)
                 if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
                     find_package(CUDAToolkit REQUIRED)
                     target_link_libraries(${a_target} PRIVATE CUDA::cudart CUDA::cufft)
-                else()
+                elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+                    target_link_libraries(${a_target} PRIVATE $ENV{CRAY_HIPFORT_LIB})
+                else() 
                     find_package(hipfort COMPONENTS hipfft CONFIG REQUIRED)
-                    target_link_libraries(${a_target} PRIVATE hipfort::hipfft)
+                    target_link_libraries(${a_target} PRIVATE hipfort::hipfft)                    
                 endif()
             else()
                 find_package(FFTW REQUIRED)
@@ -584,7 +603,7 @@ function(MFC_SETUP_TARGET)
                         PRIVATE -DFRONTIER_UNIFIED)
                 endif()
 
-    		find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
+                find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
                 target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn flang_rt.hostdevice)
             endif()
         elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")

load_amd.sh

@@ -1,7 +1,33 @@
-module use /ccs/home/bcornille/afar-drops/modulefiles/Core/
-module load rocm-afar-drop mpich cray-python
-module load cmake
-#export OMPX_APU_MAPS=0
-#export HSA_XNACK=0
-#export LIBOMPTARGET_INFO=0
-#export AMD_LOG_LEVEL=1
+module load python cmake
+module load cpe/25.09
+module load PrgEnv-amd
+
+AFAR_UMS_BASEDIR="/sw/crusher/ums/compilers/afar"
+AFAR_UMS_LATEST=$(ls -d --color=never ${AFAR_UMS_BASEDIR}/*/ | tail -n1)
+export OLCF_AFAR_ROOT=${AFAR_UMS_BASEDIR}/"rocm-afar-8873-drop-22.2.0"
+
+export PATH=${OLCF_AFAR_ROOT}/lib/llvm/bin:${PATH}
+export LD_LIBRARY_PATH=${OLCF_AFAR_ROOT}/lib:${OLCF_AFAR_ROOT}/lib/llvm/lib:${LD_LIBRARY_PATH}
+
+export CRAY_MPICH_INC="-I${OLCF_AFAR_ROOT}/include/mpich3.4a2"
+export CRAY_HIPFORT_INC="-I${OLCF_AFAR_ROOT}/include/hipfort/amdgcn"
+export CRAY_HIPFORT_LIB="-L${OLCF_AFAR_ROOT}/lib -lhipfft"
+export CRAY_HIP_INC="-I${OLCF_AFAR_ROOT}/include/hip"
+export CRAY_MPICH_LIB="-L${CRAY_MPICH_PREFIX}/lib \
+                        ${CRAY_PMI_POST_LINK_OPTS} \
+                        -lmpifort_amd -lmpi_amd -lmpi -lpmi -lpmi2"
+export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${CRAY_LD_LIBRARY_PATH}"
+export CMAKE_PREFIX_PATH="${OLCF_AFAR_ROOT}:${CMAKE_PREFIX_PATH}"
+export FC="${OLCF_AFAR_ROOT}/bin/amdflang"
+
+unset MPICH_GPU_SUPPORT_ENABLED
+
+# module use /ccs/home/bcornille/afar-drops/modulefiles/Core/
+# module load rocm-afar-drop mpich cray-python
+# module load cmake
+
+export OMPX_APU_MAPS=0
+export HSA_XNACK=0
+export LIBOMPTARGET_INFO=0
+export AMD_LOG_LEVEL=0
+export OFFLOAD_TRACK_ALLOCATION_TRACES=false

src/common/include/shared_parallel_macros.fpp

@@ -1,3 +1,13 @@
+#:set NVIDIA_COMPILER_ID="NVHPC"
+#:set PGI_COMPILER_ID="PGI"
+#:set INTEL_COMPILER_ID="Intel"
+#:set CCE_COMPILER_ID="Cray"
+#:set AMD_COMPILER_ID="LLVMFlang"
+
+#:set USING_NVHPC = (MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID)
+#:set USING_CCE = (MFC_COMPILER == CCE_COMPILER_ID)
+#:set USING_AMD = (MFC_COMPILER == AMD_COMPILER_ID)
+
 #:def ASSERT_LIST(data, datatype)
     #:assert data is not None
     #:assert isinstance(data, list)