cuD-PDLP

ci/validate_wheel.sh

@@ -22,11 +22,11 @@ PYDISTCHECK_ARGS=(
 if [[ "${package_dir}" == "python/libcuopt" ]]; then
     if [[ "${RAPIDS_CUDA_MAJOR}" == "12" ]]; then
         PYDISTCHECK_ARGS+=(
-            --max-allowed-size-compressed '690Mi'
+            --max-allowed-size-compressed '960Mi'
         )
     else
         PYDISTCHECK_ARGS+=(
-            --max-allowed-size-compressed '550Mi'
+            --max-allowed-size-compressed '800Mi'
         )
     fi
 elif [[ "${package_dir}" != "python/cuopt" ]] && \

cpp/CMakeLists.txt

@@ -311,6 +311,44 @@ create_logger_macros(CUOPT "cuopt::default_logger()" include/cuopt)
 
 find_package(CUDSS REQUIRED)
 
+# ##################################################################################################
+# - NCCL (multi-GPU distributed PDLP) -------------------------------------------------------------
+# NCCL is shipped via the conda env; no canonical CMake config target, so look it
+# up by name in the standard lib paths (plus CONDA_PREFIX as a hint).
+set(NCCL_HINT_PREFIXES "")
+if (DEFINED ENV{CONDA_PREFIX} AND NOT "$ENV{CONDA_PREFIX}" STREQUAL "")
+    list(APPEND NCCL_HINT_PREFIXES "$ENV{CONDA_PREFIX}")
+endif ()
+find_path(NCCL_INCLUDE_DIR
+    NAMES nccl.h
+    HINTS ${NCCL_HINT_PREFIXES}
+    PATH_SUFFIXES include
+)
+find_library(NCCL_LIBRARY
+    NAMES nccl
+    HINTS ${NCCL_HINT_PREFIXES}
+    PATH_SUFFIXES lib lib64
+)
+if (NOT NCCL_INCLUDE_DIR OR NOT NCCL_LIBRARY)
+    message(FATAL_ERROR "NCCL not found. Looked in ${NCCL_HINT_PREFIXES}. Install nccl-dev / libnccl-dev in the active env.")
+endif ()
+add_library(nccl_external UNKNOWN IMPORTED GLOBAL)
+set_target_properties(nccl_external PROPERTIES
+    IMPORTED_LOCATION "${NCCL_LIBRARY}"
+    INTERFACE_INCLUDE_DIRECTORIES "${NCCL_INCLUDE_DIR}"
+)
+message(STATUS "Using NCCL: ${NCCL_LIBRARY}")
+
+# ##################################################################################################
+# - KaMinPar (multi-threaded partitioning for distributed PDLP) ------------------------------------
+# Brought in the RAPIDS way (rapids_cpm_find): uses an installed KaMinPar (deb/rpm/conda,
+# discovered via its CMake config) if present, otherwise builds the pinned source via CPM.
+include(cmake/thirdparty/get_kaminpar.cmake)
+if (NOT TARGET KaMinPar::KaMinPar)
+    message(FATAL_ERROR "KaMinPar::KaMinPar was not made available by get_kaminpar.cmake")
+endif ()
+message(STATUS "Using KaMinPar (distributed PDLP graph partitioner)")
+
 # ##################################################################################################
 # - gRPC and Protobuf setup -----------------------------------------------------------------------
 
@@ -546,6 +584,32 @@ target_include_directories(cuopt
 target_link_libraries(cuopt PRIVATE $<TARGET_FILE:PSLP>)
 add_dependencies(cuopt PSLP)
 
+# Link KaMinPar by file to avoid export dependency tracking (mirrors PSLP above).
+# KaMinPar is a from-source static library fully embedded into libcuopt.so; it is never
+# installed (INSTALL_KAMINPAR OFF) and consumers of cuopt::cuopt never use it, so it must
+# not leak into cuopt's exported link interface (otherwise rapids_export fails with
+# "target KaMinPar is not in any export set"). libKaMinPar.a is self-contained (the
+# kaminpar-common OBJECT lib is archived into it); we only need its public headers
+# (<kaminpar.h>, which pulls in stdlib + TBB) at compile time.
+target_include_directories(cuopt SYSTEM PRIVATE
+        $<TARGET_PROPERTY:KaMinPar::KaMinPar,INTERFACE_INCLUDE_DIRECTORIES>)
+# partitioner.cpp includes <kaminpar.h>. Because KaMinPar is linked by file, cuopt
+# does NOT inherit KaMinPar's PUBLIC compile definitions, so we must replicate the ones that
+# affect what <kaminpar.h> compiles to:
+#   * TBB_PREVIEW_GLOBAL_CONTROL  - <kaminpar.h> includes <tbb/global_control.h>, which older
+#                                   TBB gates behind this macro.
+#   * KAMINPAR_64BIT_EDGE_IDS     - selects EdgeID = uint64 in <kaminpar.h>. MUST stay in sync
+#                                   with the KAMINPAR_64BIT_* options in get_kaminpar.cmake;
+#                                   otherwise the public API types (e.g. copy_graph's span<>
+#                                   widths) mismatch the prebuilt archive => undefined refs.
+target_compile_definitions(cuopt PRIVATE TBB_PREVIEW_GLOBAL_CONTROL KAMINPAR_64BIT_EDGE_IDS)
+target_link_libraries(cuopt PRIVATE $<TARGET_FILE:KaMinPar::KaMinPar>)
+# Real (non-imported) target only exists when KaMinPar is built from source via CPM;
+# force it to build before cuopt since it is EXCLUDE_FROM_ALL.
+if (TARGET KaMinPar)
+        add_dependencies(cuopt KaMinPar)
+endif ()
+
 # ##################################################################################################
 # - link libraries --------------------------------------------------------------------------------
 
@@ -596,6 +660,7 @@ target_link_libraries(cuopt
         ${CUDSS_LIB_FILE}
         PRIVATE
         ${CUOPT_PRIVATE_CUDA_LIBS}
+        nccl_external
         $<$<BOOL:${CUOPT_ENABLE_GRPC}>:protobuf::libprotobuf>
         $<$<BOOL:${CUOPT_ENABLE_GRPC}>:gRPC::grpc++>
 )

cpp/cmake/thirdparty/get_kaminpar.cmake

@@ -0,0 +1,78 @@
+# cmake-format: off
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# cmake-format: on
+
+# Multi-threaded graph partitioner for distributed PDLP.
+# Uses rapids_cpm_find so a system / conda / .deb install of KaMinPar (which ships a
+# CMake config package exporting KaMinPar::KaMinPar) is used when available, and
+# otherwise the pinned source is cloned and built via CPM. KaMinPar depends on TBB,
+# which cuOpt already requires (see find_package(TBB) for papilo).
+function(find_and_configure_kaminpar)
+    set(oneValueArgs VERSION PINNED_TAG)
+    cmake_parse_arguments(PKG "" "${oneValueArgs}" "" ${ARGN})
+
+    # NOTE: KaMinPar is intentionally NOT added to cuopt's BUILD/INSTALL export sets.
+    # It is a from-source static dependency that is fully embedded into libcuopt.so and
+    # never installed (INSTALL_KAMINPAR OFF below). Registering it in cuopt-exports would
+    # both break export generation ("target KaMinPar is not in any export set") and emit a
+    # bogus find_dependency(KaMinPar) into the installed cuopt config. It is linked by file
+    # in cpp/CMakeLists.txt (mirroring PSLP) so it stays out of cuopt's export interface.
+    rapids_cpm_find(KaMinPar ${PKG_VERSION}
+        GLOBAL_TARGETS KaMinPar::KaMinPar
+        CPM_ARGS
+        GIT_REPOSITORY https://github.com/KaHIP/KaMinPar.git
+        GIT_TAG ${PKG_PINNED_TAG}
+        EXCLUDE_FROM_ALL
+        OPTIONS
+            "KAMINPAR_BUILD_APPS OFF"
+            "KAMINPAR_BUILD_TOOLS OFF"
+            "KAMINPAR_BUILD_TESTS OFF"
+            "KAMINPAR_BUILD_BENCHMARKS OFF"
+            "KAMINPAR_BUILD_EXAMPLES OFF"
+            "KAMINPAR_BUILD_DISTRIBUTED OFF"
+            # Timers use global state and force single-threaded use of the library
+            # interface; disable so cuOpt can call the partitioner freely.
+            "KAMINPAR_ENABLE_TIMERS OFF"
+            # Avoid an extra hard dependency on Google Sparsehash.
+            "KAMINPAR_BUILD_WITH_SPARSEHASH OFF"
+            # cuOpt's TBB is discovered via a legacy find that only exposes TBB::tbb
+            # (no TBB::tbbmalloc target); disable KaMinPar's optional tbbmalloc use.
+            "KAMINPAR_ENABLE_TBB_MALLOC OFF"
+            # Large LP constraint graphs can exceed 2^31 directed edges.
+            "KAMINPAR_64BIT_EDGE_IDS ON"
+            "INSTALL_KAMINPAR OFF"
+            # Build KaMinPar as a STATIC library that is embedded into libcuopt.so (linked
+            # by file in cpp/CMakeLists.txt). The wheel build configures with
+            # BUILD_SHARED_LIBS=ON; without this override KaMinPar would build a separate
+            # libKaMinPar.so that is neither embedded nor shipped in the wheel. Forcing PIC
+            # is required so the static objects can be linked into the shared libcuopt.so
+            # (KaMinPar's KaMinParCommon OBJECT lib otherwise lacks -fPIC).
+            "BUILD_SHARED_LIBS OFF"
+            "CMAKE_POSITION_INDEPENDENT_CODE ON"
+    )
+
+    if(KaMinPar_ADDED)
+        message(VERBOSE "CUOPT: Using KaMinPar located in ${KaMinPar_SOURCE_DIR}")
+        # KaMinPar's public header pulls in <tbb/global_control.h>. On older TBB releases
+        # that header is gated behind TBB_PREVIEW_GLOBAL_CONTROL (KaMinPar upstream assumes a
+        # newer oneTBB and never defines it). Define it on KaMinParCommon PUBLIC so it
+        # propagates to all KaMinPar translation units (KaMinPar links KaMinParCommon PUBLIC).
+        # Harmless on newer oneTBB where global_control is no longer a preview feature.
+        # Also force PIC on every KaMinPar target (the KaMinParCommon OBJECT library does not
+        # reliably inherit CMAKE_POSITION_INDEPENDENT_CODE) so the static archive can be
+        # embedded into the shared libcuopt.so.
+        foreach(_kaminpar_tgt KaMinParCommon KaMinPar KaMinParIO)
+            if(TARGET ${_kaminpar_tgt})
+                set_target_properties(${_kaminpar_tgt} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+            endif()
+        endforeach()
+        if(TARGET KaMinParCommon)
+            target_compile_definitions(KaMinParCommon PUBLIC TBB_PREVIEW_GLOBAL_CONTROL)
+        endif()
+    else()
+        message(VERBOSE "CUOPT: Using KaMinPar located in ${KaMinPar_DIR}")
+    endif()
+endfunction()
+
+find_and_configure_kaminpar(VERSION 3.7.3 PINNED_TAG v3.7.3)

cpp/cuopt_cli.cpp

@@ -5,6 +5,7 @@
  */
 /* clang-format on */
 
+#include <cuopt/error.hpp>
 #include <cuopt/linear_programming/backend_selection.hpp>
 #include <cuopt/linear_programming/cpu_optimization_problem.hpp>
 #include <cuopt/linear_programming/io/parser.hpp>
@@ -176,7 +177,17 @@ int run_single_file(const std::string& file_path,
       auto solution = cuopt::linear_programming::solve_mip(problem_interface.get(), mip_settings);
     } else {
       auto& lp_settings = settings.get_pdlp_settings();
-      auto solution     = cuopt::linear_programming::solve_lp(problem_interface.get(), lp_settings);
+
+      if (lp_settings.hyper_params.use_distributed_pdlp) {
+        cuopt::cuopt_expects(
+          handle_ptr != nullptr,
+          cuopt::error_type_t::ValidationError,
+          "Distributed PDLP requires the GPU memory backend; no GPU handle is available for the "
+          "selected memory backend.");
+        cuopt::linear_programming::solve_lp(handle_ptr.get(), mps_data_model, lp_settings);
+      } else {
+        cuopt::linear_programming::solve_lp(problem_interface.get(), lp_settings);
+      }
     }
   } catch (const std::exception& e) {
     fprintf(stderr, "cuopt_cli error: %s\n", e.what());
@@ -421,10 +432,20 @@ int main(int argc, char* argv[])
   std::vector<rmm::mr::cuda_async_memory_resource> memory_resources;
 
   if (memory_backend == cuopt::linear_programming::memory_backend_t::GPU) {
-    const int num_gpus = settings.get_parameter<int>(CUOPT_NUM_GPUS);
+    // Get the right number of GPUs
+    // Distributed PDLP uses its own knob: distributed_pdlp_num_gpus
+    // Everything else uses num_gpus which is capped at 2
+    const bool use_distributed_pdlp = settings.get_parameter<bool>(CUOPT_USE_DISTRIBUTED_PDLP);
+    int requested_gpus              = use_distributed_pdlp
+                                        ? settings.get_parameter<int>(CUOPT_DISTRIBUTED_PDLP_NUM_GPUS)
+                                        : settings.get_parameter<int>(CUOPT_NUM_GPUS);
+    if (use_distributed_pdlp && requested_gpus == -1) {
+      requested_gpus = raft::device_setter::get_device_count();
+    }
+    const int provisioned_gpus = std::min(raft::device_setter::get_device_count(), requested_gpus);
 
-    memory_resources.reserve(std::min(raft::device_setter::get_device_count(), num_gpus));
-    for (int i = 0; i < std::min(raft::device_setter::get_device_count(), num_gpus); ++i) {
+    memory_resources.reserve(provisioned_gpus);
+    for (int i = 0; i < provisioned_gpus; ++i) {
       RAFT_CUDA_TRY(cudaSetDevice(i));
       memory_resources.emplace_back();
       rmm::mr::set_per_device_resource(rmm::cuda_device_id{i}, memory_resources.back());

cpp/include/cuopt/linear_programming/constants.h

@@ -82,14 +82,20 @@
 #define CUOPT_MIP_STRONG_BRANCHING_SIMPLEX_ITERATION_LIMIT \
   "mip_strong_branching_simplex_iteration_limit"
 
-#define CUOPT_SOLUTION_FILE            "solution_file"
-#define CUOPT_NUM_CPU_THREADS          "num_cpu_threads"
-#define CUOPT_NUM_GPUS                 "num_gpus"
-#define CUOPT_USER_PROBLEM_FILE        "user_problem_file"
-#define CUOPT_PRESOLVE_FILE            "presolve_file"
-#define CUOPT_RANDOM_SEED              "random_seed"
-#define CUOPT_PDLP_PRECISION           "pdlp_precision"
-#define CUOPT_MIP_SEMICONTINUOUS_BIG_M "mip_semi_continuous_big_m"
+#define CUOPT_SOLUTION_FILE                   "solution_file"
+#define CUOPT_NUM_CPU_THREADS                 "num_cpu_threads"
+#define CUOPT_NUM_GPUS                        "num_gpus"
+#define CUOPT_DISTRIBUTED_PDLP_NUM_GPUS       "distributed_pdlp_num_gpus"
+#define CUOPT_MULTI_GPU_PARTITION_FILE        "multi_gpu_partition_file"
+#define CUOPT_MULTI_GPU_EXPORT_PARTITION_FILE "multi_gpu_export_partition_file"
+#define CUOPT_DISTRIBUTED_PDLP_PARTITIONER    "distributed_pdlp_partitioner"
+#define CUOPT_USE_DISTRIBUTED_PDLP            "use_distributed_pdlp"
+#define CUOPT_PDLP_DISABLE_GRAPH              "pdlp_disable_graph"
+#define CUOPT_USER_PROBLEM_FILE               "user_problem_file"
+#define CUOPT_PRESOLVE_FILE                   "presolve_file"
+#define CUOPT_RANDOM_SEED                     "random_seed"
+#define CUOPT_PDLP_PRECISION                  "pdlp_precision"
+#define CUOPT_MIP_SEMICONTINUOUS_BIG_M        "mip_semi_continuous_big_m"
 
 #define CUOPT_MIP_HYPER_HEURISTIC_POPULATION_SIZE     "mip_hyper_heuristic_population_size"
 #define CUOPT_MIP_HYPER_HEURISTIC_NUM_CPUFJ_THREADS   "mip_hyper_heuristic_num_cpufj_threads"

cpp/include/cuopt/linear_programming/pdlp/pdlp_hyper_params.cuh

@@ -47,12 +47,17 @@ struct pdlp_hyper_params_t {
   bool bound_objective_rescaling                                  = true;
   bool use_reflected_primal_dual                                  = true;
   bool use_fixed_point_error                                      = true;
-  double reflection_coefficient                                   = 1.0;
-  double restart_k_p                                              = 0.99;
-  double restart_k_i                                              = 0.01;
-  double restart_k_d                                              = 0.0;
-  double restart_i_smooth                                         = 0.3;
-  bool use_conditional_major                                      = true;
+  bool use_distributed_pdlp                                       = false;
+  // Debug/diagnostic knob: when true, PDLP bypasses CUDA-graph capture in
+  // ping_pong_graph_t and executes each iteration eagerly
+  // Useful for cuD-PDLP in current state
+  bool pdlp_disable_graph       = false;
+  double reflection_coefficient = 1.0;
+  double restart_k_p            = 0.99;
+  double restart_k_i            = 0.01;
+  double restart_k_d            = 0.0;
+  double restart_i_smooth       = 0.3;
+  bool use_conditional_major    = true;
 };
 
 // TODO most likely we want to get rid of pdlp_solver_mode and just have prebuilt

cpp/include/cuopt/linear_programming/pdlp/solver_settings.hpp

@@ -307,6 +307,21 @@ class pdlp_solver_settings_t {
   presolver_t presolver{presolver_t::Default};
   bool dual_postsolve{true};
   int num_gpus{1};
+  // Number of GPUs to use specifically for distributed PDLP (use_distributed_pdlp=true).
+  // -1 means auto-detect
+  int distributed_pdlp_num_gpus{-1};
+  std::string multi_gpu_partition_file{""};
+  // If non-empty, the partition computed for distributed PDLP is written to this
+  // path (one part-id per line) right after partitioning. The file can be fed
+  // back via multi_gpu_partition_file.
+  std::string multi_gpu_export_partition_file{""};
+  // Which graph partitioner distributed PDLP uses. One of:
+  //   "auto"     - 1 GPU => Dummy; otherwise KaMinPar
+  //   "dummy"    - round-robin, no graph (trivial)
+  //   "kaminpar" - multi-threaded KaMinPar
+  std::string distributed_pdlp_partitioner{"auto"};
+  // Set to true inside the shards
+  bool is_distributed_sub_pdlp{false};
   method_t method{method_t::Concurrent};
   bool inside_mip{false};
   // For concurrent termination

cpp/src/math_optimization/solver_settings.cu

@@ -144,6 +144,7 @@ solver_settings_t<i_t, f_t>::solver_settings_t() : pdlp_settings(), mip_settings
     {CUOPT_MIP_OBJECTIVE_STEP, &mip_settings.objective_step, 0, 1, 1},
     {CUOPT_NUM_GPUS, &pdlp_settings.num_gpus, 1, 2, 1},
     {CUOPT_NUM_GPUS, &mip_settings.num_gpus, 1, 2, 1},
+    {CUOPT_DISTRIBUTED_PDLP_NUM_GPUS, &pdlp_settings.distributed_pdlp_num_gpus, -1, 576, -1},
     {CUOPT_MIP_BATCH_PDLP_STRONG_BRANCHING, &mip_settings.mip_batch_pdlp_strong_branching, 0, 2, 0},
     {CUOPT_MIP_BATCH_PDLP_RELIABILITY_BRANCHING, &mip_settings.mip_batch_pdlp_reliability_branching, 0, 2, 0},
     {CUOPT_MIP_STRONG_BRANCHING_SIMPLEX_ITERATION_LIMIT, &mip_settings.strong_branching_simplex_iteration_limit, -1,std::numeric_limits<i_t>::max(), -1},
@@ -191,6 +192,8 @@ solver_settings_t<i_t, f_t>::solver_settings_t() : pdlp_settings(), mip_settings
     {CUOPT_DUAL_POSTSOLVE, &pdlp_settings.dual_postsolve, true},
     {CUOPT_BARRIER_ITERATIVE_REFINEMENT, &pdlp_settings.barrier_iterative_refinement, true},
     {CUOPT_MIP_PROBING, &mip_settings.probing, true},
+    {CUOPT_USE_DISTRIBUTED_PDLP, &pdlp_settings.hyper_params.use_distributed_pdlp, false},
+    {CUOPT_PDLP_DISABLE_GRAPH, &pdlp_settings.hyper_params.pdlp_disable_graph, false},
     // Diving heuristic hyper-parameters (hidden from default --help: name contains "hyper_")
     {CUOPT_MIP_HYPER_DIVING_SHOW_TYPE, &mip_settings.diving_params.show_type, false, "log diving heuristic type when it finds a new incumbent"},
   };
@@ -203,7 +206,10 @@ solver_settings_t<i_t, f_t>::solver_settings_t() : pdlp_settings(), mip_settings
     {CUOPT_USER_PROBLEM_FILE, &mip_settings.user_problem_file, ""},
     {CUOPT_USER_PROBLEM_FILE, &pdlp_settings.user_problem_file, ""},
     {CUOPT_PRESOLVE_FILE, &mip_settings.presolve_file, ""},
-    {CUOPT_PRESOLVE_FILE, &pdlp_settings.presolve_file, ""}
+    {CUOPT_PRESOLVE_FILE, &pdlp_settings.presolve_file, ""},
+    {CUOPT_MULTI_GPU_PARTITION_FILE, &pdlp_settings.multi_gpu_partition_file, ""},
+    {CUOPT_MULTI_GPU_EXPORT_PARTITION_FILE, &pdlp_settings.multi_gpu_export_partition_file, ""},
+    {CUOPT_DISTRIBUTED_PDLP_PARTITIONER, &pdlp_settings.distributed_pdlp_partitioner, "auto"},
   };
   // clang-format on
 }

cpp/src/pdlp/CMakeLists.txt

@@ -29,6 +29,11 @@ set(LP_CORE_FILES
   ${CMAKE_CURRENT_SOURCE_DIR}/termination_strategy/convergence_information.cu
   ${CMAKE_CURRENT_SOURCE_DIR}/optimal_batch_size_handler/optimal_batch_size_handler.cu
   ${CMAKE_CURRENT_SOURCE_DIR}/utilities/ping_pong_graph.cu
+  ${CMAKE_CURRENT_SOURCE_DIR}/distributed_pdlp/shard.cu
+  ${CMAKE_CURRENT_SOURCE_DIR}/distributed_pdlp/multi_gpu_engine.cu
+  ${CMAKE_CURRENT_SOURCE_DIR}/distributed_pdlp/distributed_algorithms.cu
+  ${CMAKE_CURRENT_SOURCE_DIR}/distributed_pdlp/partition_loader.cu
+  ${CMAKE_CURRENT_SOURCE_DIR}/distributed_pdlp/partitioner.cpp
 )
 
 # C and Python adapter files