Merge branch 'ngt-ci-pipeline' into template

Author: OliverRietmann

.github/scripts/csv_to_md.py

@@ -38,10 +38,13 @@ def get_emoji(t):
   baseline_name, baseline_mean, baseline_stdev, count_baseline = baseline
   name, mean, stdev, count = current
   assert(baseline_name == name)
+  total_time_baseline = baseline_mean * (count_baseline // args.runs)
   total_time = mean * (count // args.runs)
   d, s, t = student(baseline_mean, baseline_stdev, args.runs, mean, stdev, args.runs)
   emoji = get_emoji(t)
-  table.append([name, int(total_time), int(mean), f'{stdev:.2f}', f'{d:.2f}', f'{t:.2f}', emoji])
+  table.append([name, int(total_time), int(mean), f'{stdev:.2f}', count, int(total_time_baseline), int(baseline_mean), f'{baseline_stdev:.2f}', count_baseline, f'{d:.2f}', f'{s:.2f}', f'{t:.2f}', emoji])
 
-header = ['name', 'total time (\u03BCs)', 'mean (\u03BCs)', 'stdev \u03C3', 'diff \u0394', 't', '']
+table.sort(key = lambda row: row[1], reverse=True)
+
+header = ['name', 'total time (\u03BCs)', 'mean (\u03BCs)', 'stdev \u03C3', 'samples', 'total time (base)', 'mean (base)', 'stdev (base)', 'samples (base)', 'diff \u0394', 'stdev (pooled)', 't', '']
 print(tab.tabulate(table, header, tablefmt="github"))

.github/workflows/standalone-benchmark.yml

@@ -17,7 +17,7 @@ jobs:
         name: [cpu, nvidia-h100, nvidia-l40s, amd-mi300x, amd-w7900]
         include:
           - name: cpu
-            runner: cern-nextgen-h100
+            runner: cern-nextgen-mi300x
             cmake_args: -DENABLE_CUDA=0 -DENABLE_HIP=0
             profiler_runs: 42
             standalone_runs: 42
@@ -53,7 +53,7 @@ jobs:
       STANDALONE_DIR: /root/standalone
       BENCHMARK_CSV: standalone_${{ matrix.name }}.csv
       PROFILER_CSV: profiler_${{ matrix.name }}.csv
-      TIMING_CA: ./ca -e 50kHz ${{ matrix.cpu_gpu }} --seed 0 --sync --debug 1 # Add --PROCdebugMarkdown 1 --runs 42 --runsInit 2 --PROCresetTimers 1 for benchmark runs
+      TIMING_CA: ./ca -e 50kHz ${{ matrix.cpu_gpu }} --seed 0 --sync --runsInit 0 --PROCresetTimers 1 --PROCdebugMarkdown 1 --debug 1 # Add --runs 42 for benchmark runs
       LD_LIBRARY_PATH: /usr/local/cuda-13.0/compat
 
     name: ${{ matrix.name }}
@@ -104,7 +104,7 @@ jobs:
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
-          ${TIMING_CA} --debug 1 --runs ${{ matrix.standalone_runs }} --runsInit 0 --PROCdebugMarkdown 1 --PROCresetTimers 1 --PROCdebugCSV /root/${BENCHMARK_CSV}
+          ${TIMING_CA} --runs ${{ matrix.standalone_runs }} --PROCdebugCSV /root/${BENCHMARK_CSV}
           python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_standalone.py --discard 0 --input /root/${BENCHMARK_CSV} --output /root/summary_${BENCHMARK_CSV}
       
       - name: Profiler - Nsight Compute
@@ -114,7 +114,7 @@ jobs:
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
-          ncu --set none --metrics gpu__time_duration.avg --export ${{ matrix.name }} --clock-control none --force-overwrite ${TIMING_CA} --runs ${{ matrix.profiler_runs }} --debug 1 --PROCdebugMarkdown 1  # Generates ${{ matrix.name }}.ncu-rep
+          ncu --set none --metrics gpu__time_duration.avg --export ${{ matrix.name }} --clock-control none --force-overwrite ${TIMING_CA} --runs ${{ matrix.profiler_runs }}  # Generates ${{ matrix.name }}.ncu-rep
           ncu --import ${STANDALONE_DIR}/${{ matrix.name }}.ncu-rep --print-units base --csv > /root/${PROFILER_CSV}
           rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
           python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_ncu.py --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
@@ -127,7 +127,7 @@ jobs:
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
-          nsys profile -o ${{ matrix.name }} ${TIMING_CA} --runs ${{ matrix.profiler_runs }} --debug 1 --PROCdebugMarkdown 1  # Generates ${{ matrix.name }}.nsys-rep
+          nsys profile -o ${{ matrix.name }} ${TIMING_CA} --runs ${{ matrix.profiler_runs }}  # Generates ${{ matrix.name }}.nsys-rep
           nsys stats --report cuda_gpu_kern_sum --timeunit usec --force-export=true --format csv ${{ matrix.name }}.nsys-rep > /root/${PROFILER_CSV}
           rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
           python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_nsys.py --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
@@ -138,7 +138,7 @@ jobs:
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
-          rocprofv2 --output-directory /root --output-file-name ${{ matrix.name }} ${TIMING_CA} --runs ${{ matrix.standalone_runs }} --debug 1 --PROCdebugMarkdown 1  # Generates results_${{ matrix.name }}.csv
+          rocprofv2 --output-directory /root --output-file-name ${{ matrix.name }} ${TIMING_CA} --runs ${{ matrix.standalone_runs }}  # Generates results_${{ matrix.name }}.csv
           rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
           mv /root/results_${{ matrix.name }}.csv /root/${PROFILER_CSV}
           python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_rocprofv2.py --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}