GPU CMake: Write separate headers for GPU device and non-device parameters

davidrohr · davidrohr · commit 51818cbfaa5f · 2026-02-08T00:06:10.000+01:00
diff --git a/GPU/GPUTracking/Base/cuda/CMakeLists.txt b/GPU/GPUTracking/Base/cuda/CMakeLists.txt
@@ -74,7 +74,7 @@ add_custom_command(
     COMMAND cat ${GPUDIR}/Base/GPUStdSystemHeaders.h >> ${GPU_RTC_BIN}.src
     COMMAND ${CMAKE_CUDA_COMPILER} ${GPU_RTC_DEFINES} ${GPU_RTC_INCLUDES} -std=c++${CMAKE_CUDA_STANDARD} -D__CUDA_ARCH__=${RTC_CUDA_ARCH} -Wno-deprecated-gpu-targets -D__CUDACC__ -x c++ -M -MD -MT ${GPU_RTC_BIN}.src -MF ${GPU_RTC_BIN}.src.d ${GPU_RTC_SRC}
     COMMAND ${CMAKE_CUDA_COMPILER} ${GPU_RTC_DEFINES} ${GPU_RTC_INCLUDES} -std=c++${CMAKE_CUDA_STANDARD} -D__CUDA_ARCH__=${RTC_CUDA_ARCH} -Wno-deprecated-gpu-targets -D__CUDACC__ -x c++ -E -Xcompiler "-nostdinc -P" ${GPU_RTC_SRC} >> ${GPU_RTC_BIN}.src
-    DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/cuda/GPUReconstructionCUDAIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h GPU_PARAM_HEADER_AUTO_ALL
+    DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/cuda/GPUReconstructionCUDAIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h GPU_PARAM_HEADER_TARGET
     DEPFILE ${GPU_RTC_BIN}.src.d
     COMMAND_EXPAND_LISTS
     COMMENT "Preparing CUDA RTC source file ${GPU_RTC_BIN}.src"
diff --git a/GPU/GPUTracking/Base/hip/CMakeLists.txt b/GPU/GPUTracking/Base/hip/CMakeLists.txt
@@ -125,7 +125,7 @@ add_custom_command(
     COMMAND cat ${GPUDIR}/Base/hip/GPUReconstructionHIPIncludesSystem.h | grep -v GPUStdSystemHeaders.h >> ${GPU_RTC_BIN}.src
     COMMAND cat ${GPUDIR}/Base/GPUStdSystemHeaders.h >> ${GPU_RTC_BIN}.src
     COMMAND ${CMAKE_HIP_COMPILER} ${GPU_RTC_DEFINES} ${GPU_RTC_INCLUDES} -std=c++${CMAKE_HIP_STANDARD} -D__HIPCC__ -D__HIP_DEVICE_COMPILE__ -x c++ -nostdinc -E -P ${GPU_RTC_SRC} -MD -MT ${GPU_RTC_BIN}.src -MF ${GPU_RTC_BIN}.src.d >> ${GPU_RTC_BIN}.src
-    DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/hip/GPUReconstructionHIPIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h ${MODULE}_HIPIFIED GPU_PARAM_HEADER_AUTO_ALL
+    DEPENDS ${GPU_RTC_SRC} ${GPUDIR}/Base/GPUStdSystemHeaders.h ${GPUDIR}/Base/hip/GPUReconstructionHIPIncludesSystem.h ${GPUDIR}/Base/GPUStdSystemHeaders.h ${MODULE}_HIPIFIED GPU_PARAM_HEADER_TARGET
     DEPFILE ${GPU_RTC_BIN}.src.d
     COMMAND_EXPAND_LISTS
     COMMENT "Preparing HIP RTC source file ${GPU_RTC_BIN}.src"
diff --git a/GPU/GPUTracking/CMakeLists.txt b/GPU/GPUTracking/CMakeLists.txt
@@ -130,7 +130,8 @@ set(ON_THE_FLY_DIR ${CMAKE_CURRENT_BINARY_DIR}/include_gpu_onthefly)
 file(MAKE_DIRECTORY ${ON_THE_FLY_DIR})
 include(cmake/gpu_param_header_generator.cmake)
 set(GPU_DEFAULT_PARAMS_HEADER ${ON_THE_FLY_DIR}/GPUDefParametersDefaults.h)
-generate_gpu_param_header("AUTO" ${GPU_DEFAULT_PARAMS_HEADER}) # generate header with default GPU parameters, arch selected by CMake variables
+set(GPU_DEFAULT_PARAMS_HEADER_DEVICE ${ON_THE_FLY_DIR}/GPUDefParametersDefaultsDevice.h)
+generate_gpu_param_header("ALL" ${GPU_DEFAULT_PARAMS_HEADER} ${GPU_DEFAULT_PARAMS_HEADER_DEVICE} GPU_CONST_PARAM_ARCHITECTUES) # generate header with default GPU parameters, arch selected by CMake variables
 
 set(HDRS_INSTALL
     ${HDRS_CINT_O2}
@@ -161,6 +162,7 @@ set(HDRS_INSTALL
     Debug/GPUROOTDump.h
     Definitions/GPUDefConstantsAndSettings.h
     ${GPU_DEFAULT_PARAMS_HEADER}
+    ${GPU_DEFAULT_PARAMS_HEADER_DEVICE}
     Definitions/GPUDefParametersWrapper.h
     Definitions/GPUDefParametersConstants.h
     Definitions/GPUDef.h
@@ -449,20 +451,19 @@ if(CUDA_ENABLED OR OPENCL_ENABLED OR HIP_ENABLED)
     message(WARNING "GPU Tracking disabled on MacOS")
   else()
     make_directory(${CMAKE_CURRENT_BINARY_DIR}/genGPUArch)
-    set(GPU_CONST_PARAM_FILES "")
-    set(GPU_ARCH_PARAMS_HEADER ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/GPUDefParametersDefaults_OnTheFly.h)
-    generate_gpu_param_header("ALL" ${GPU_ARCH_PARAMS_HEADER} "GPU_CONST_PARAM_ARCHITECTUES")
+    set(GPU_CONST_PARAM_FILES)
     foreach(GPU_ARCH ${GPU_CONST_PARAM_ARCHITECTUES})
       set(PARAMFILE ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch/gpu_const_param_${GPU_ARCH}.par)
       add_custom_command(
         OUTPUT ${PARAMFILE}
         COMMAND bash -c
-                "echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"${GPU_ARCH_PARAMS_HEADER}\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'"
+                "echo -e '#define GPUCA_GPUTYPE_${GPU_ARCH}\\n#define PARAMETER_FILE \"GPUDefParametersDefaults.h\"\\ngInterpreter->AddIncludePath(\"${CMAKE_CURRENT_SOURCE_DIR}/Definitions\");\\ngInterpreter->AddIncludePath(\"${ON_THE_FLY_DIR}\");\\n.x ${CMAKE_CURRENT_SOURCE_DIR}/Standalone/tools/dumpGPUDefParam.C(\"${PARAMFILE}\")\\n.q\\n'"
                 | root -l -b > /dev/null
         VERBATIM
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/genGPUArch
         MAIN_DEPENDENCY Standalone/tools/dumpGPUDefParam.C
-        DEPENDS ${GPU_ARCH_PARAMS_HEADER}
+        DEPENDS ${GPU_DEFAULT_PARAMS_HEADER}
+                ${GPU_DEFAULT_PARAMS_HEADER_DEVICE}
                 ${ON_THE_FLY_DIR}/GPUDefParametersLoadPrepare.h
                 ${ON_THE_FLY_DIR}/GPUDefParametersLoad.inc
         COMMENT "Generating GPU parameter set for architecture ${GPU_ARCH}")
diff --git a/GPU/GPUTracking/cmake/gpu_param_header_generator.cmake b/GPU/GPUTracking/cmake/gpu_param_header_generator.cmake
@@ -12,17 +12,30 @@
 # file gpu_param_header_generator.cmake
 # author Gabriele Cimador
 
-function(generate_macros json_content output types arch_list arch_list_output)
-    foreach(arch IN LISTS arch_list)
+function(generate_gpu_param_header GPU_ARCH OUT_HEADER OUT_HEADER_DEVICE)
+    if(GPU_ARCH STREQUAL "AUTO")
+        detect_gpu_arch("ALL")
+    else()
+        set(TARGET_ARCH ${GPU_ARCH})
+    endif()
+    file(READ "${GPU_PARAM_JSON}" JSON_CONTENT)
+    string(REPLACE "," ";" ARCH_LIST "${TARGET_ARCH}")
+
+    # Types
+    set(TYPES CORE LB PAR)
+    set(ARCH_LIST_EXT "${ARCH_LIST};default;default_cpu")
+    list(FIND ARCH_LIST "ALL" do_all_architectures)
+    # Per architecture definitions
+    foreach(arch IN LISTS ARCH_LIST_EXT)
         set(OUTPUT_TMP_${arch} "")
     endforeach()
-    set(arch_list_output_tmp)
-    list(FIND arch_list "ALL" do_all_architectures)
-    foreach(TYPE IN LISTS types)
-        string(JSON n_params LENGTH "${json_content}" "${TYPE}")
+    set(JSON_ARCHITECTURES)
+
+    foreach(TYPE IN LISTS TYPES)
+        string(JSON n_params LENGTH "${JSON_CONTENT}" "${TYPE}")
         math(EXPR last "${n_params} - 1")
         foreach(i RANGE 0 ${last})
-            string(JSON param_name MEMBER "${json_content}" "${TYPE}" "${i}")
+            string(JSON param_name MEMBER "${JSON_CONTENT}" "${TYPE}" "${i}")
             string(JSON n_archs LENGTH "${JSON_CONTENT}" "${TYPE}" "${param_name}")
             math(EXPR last_arch "${n_archs} - 1")
             foreach(iArch RANGE 0 ${last_arch})
@@ -31,12 +44,12 @@ function(generate_macros json_content output types arch_list arch_list_output)
                   message(FATAL_ERROR "Bogus entry ${param_name} for ${arch}")
                 endif()
                 if(do_all_architectures GREATER -1)
-                    if(arch_list_output AND NOT arch MATCHES ^default)
-                        list(APPEND arch_list_output_tmp "${arch}")
+                    if(NOT arch MATCHES ^default)
+                        list(APPEND JSON_ARCHITECTURES "${arch}")
                     endif()
                     set(list_idx 0)
                 else()
-                    list(FIND arch_list "${arch}" list_idx)
+                    list(FIND ARCH_LIST_EXT "${arch}" list_idx)
                 endif()
                 if(list_idx GREATER -1)
                     string(JSON param_values GET "${JSON_CONTENT}" "${TYPE}" "${param_name}" "${arch}")
@@ -62,60 +75,47 @@ function(generate_macros json_content output types arch_list arch_list_output)
             endforeach()
         endforeach()
     endforeach()
-    foreach(arch IN LISTS arch_list)
-        set(${output}_${arch} "${OUTPUT_TMP_${arch}}" PARENT_SCOPE)
-    endforeach()
-    if(arch_list_output)
-        list(REMOVE_DUPLICATES arch_list_output_tmp)
-        list(SORT arch_list_output_tmp)
-        set(${arch_list_output} "${arch_list_output_tmp}" PARENT_SCOPE)
-    endif()
-endfunction()
 
-function(generate_gpu_param_header GPU_ARCH OUT_HEADER)
-    set(TARGET_ARCH "UNKNOWN")
-    if(GPU_ARCH STREQUAL "AUTO")
-        detect_gpu_arch("ALL")
-    else()
-        set(TARGET_ARCH ${GPU_ARCH})
+    list(REMOVE_DUPLICATES JSON_ARCHITECTURES)
+    list(SORT JSON_ARCHITECTURES)
+    if(ARGC GREATER 3)
+        set(${ARGV3} "${JSON_ARCHITECTURES}" PARENT_SCOPE)
     endif()
-    file(READ "${GPU_PARAM_JSON}" JSON_CONTENT)
+    if(do_all_architectures GREATER -1)
+        set(ARCH_LIST ${JSON_ARCHITECTURES})
+    endif()
+
+    get_filename_component(DEVICE_HEADER_FILE "${OUT_HEADER_DEVICE}" NAME)
+
     set(TMP_HEADER "#ifndef GPUDEFPARAMETERSDEFAULTS_H\n#define GPUDEFPARAMETERSDEFAULTS_H\n\n")
+    set(TMP_HEADER_DEVICE "#ifndef GPUDEFPARAMETERSDEFAULTSDEVICE_H\n#define GPUDEFPARAMETERSDEFAULTSDEVICE_H\n\n")
     string(APPEND TMP_HEADER "// This file is auto-generated from gpu_params.json. Do not edit directly.\n")
-    string(REPLACE "," ";" ARCH_LIST "${TARGET_ARCH}")
-    string(APPEND TMP_HEADER "// Architectures: ${TARGET_ARCH}\n\n")
+    string(APPEND TMP_HEADER_DEVICE "// This file is auto-generated from gpu_params.json. Do not edit directly.\n")
+    string(APPEND TMP_HEADER_DEVICE "// Architectures: ${TARGET_ARCH}\n\n")
     string(APPEND TMP_HEADER "#if defined(GPUCA_GPUCODE) && !defined(GPUCA_GPUCODE_GENRTC) && !defined(GPUCA_GPUCODE_NO_LAUNCH_BOUNDS) // Avoid including for RTC generation besides normal include protection.\n\n")
+    string(APPEND TMP_HEADER "#include \"${DEVICE_HEADER_FILE}\"\n")
 
-    # Types
-    set(TYPES CORE LB PAR)
-    # Per architecture definitions
-    generate_macros("${JSON_CONTENT}" TMP_OUTPUT "${TYPES}" "${ARCH_LIST};default;default_cpu" "JSON_ARCHITECTURES")
-    list(FIND ARCH_LIST "ALL" do_all_architectures)
-    if(ARGC GREATER 2)
-        set(${ARGV2} "${JSON_ARCHITECTURES}" PARENT_SCOPE)
-    endif()
-    if(do_all_architectures GREATER -1)
-        set(ARCH_LIST ${JSON_ARCHITECTURES})
-    endif()
-    string(APPEND TMP_HEADER "#if 0\n")
+    string(APPEND TMP_HEADER_DEVICE "#if 0\n")
     foreach(ARCH IN LISTS ARCH_LIST)
-        string(APPEND TMP_HEADER "\n#elif defined(GPUCA_GPUTYPE_${ARCH})\n")
-        string(APPEND TMP_HEADER ${TMP_OUTPUT_${ARCH}})
+        string(APPEND TMP_HEADER_DEVICE "\n#elif defined(GPUCA_GPUTYPE_${ARCH})\n")
+        string(APPEND TMP_HEADER_DEVICE ${OUTPUT_TMP_${ARCH}})
     endforeach()
-    string(APPEND TMP_HEADER "#else\n#error GPU TYPE NOT SET\n#endif\n")
+    string(APPEND TMP_HEADER_DEVICE "#else\n#error GPU TYPE NOT SET\n#endif\n")
 
     # Default parameters
     string(APPEND TMP_HEADER "\n// Default parameters if not defined for the target architecture\n\n")
-    string(APPEND TMP_HEADER ${TMP_OUTPUT_default})
+    string(APPEND TMP_HEADER ${OUTPUT_TMP_default})
     string(APPEND TMP_HEADER "#endif // defined(GPUCA_GPUCODE) && !defined(GPUCA_GPUCODE_GENRTC) && !defined(GPUCA_GPUCODE_NO_LAUNCH_BOUNDS)\n\n")
 
     # CPU fallback
     string(APPEND TMP_HEADER "#ifndef GPUCA_GPUCODE_GENRTC // Defaults for non-LB parameters also for CPU fallback\n\n")
-    string(APPEND TMP_HEADER ${TMP_OUTPUT_default_cpu})
+    string(APPEND TMP_HEADER ${OUTPUT_TMP_default_cpu})
     string(APPEND TMP_HEADER "\n#endif // GPUCA_GPUCODE_GENRTC\n")
 
     string(APPEND TMP_HEADER "\n#endif // GPUDEFPARAMETERSDEFAULTS_H\n")
+    string(APPEND TMP_HEADER_DEVICE "\n#endif // GPUDEFPARAMETERSDEFAULTSDEVICE_H\n")
     file(GENERATE OUTPUT "${OUT_HEADER}" CONTENT "${TMP_HEADER}")
-    message(STATUS "Generated ${OUT_HEADER}")
-    add_custom_target(GPU_PARAM_HEADER_${GPU_ARCH}_ALL ALL DEPENDS ${OUT_HEADER} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/gpu_param_header_generator.cmake ${GPU_PARAM_JSON})
+    file(GENERATE OUTPUT "${OUT_HEADER_DEVICE}" CONTENT "${TMP_HEADER_DEVICE}")
+    message(STATUS "Generated ${OUT_HEADER} and ${OUT_HEADER_DEVICE}")
+    add_custom_target(GPU_PARAM_HEADER_TARGET ALL DEPENDS ${OUT_HEADER} ${OUT_HEADER_DEVICE} ${GPU_PARAM_JSON})
 endfunction()
