From 933d38c5301a06944ed748706f8b3b2a0b334c94 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 6 Aug 2026 09:41:56 -0500 Subject: [PATCH 1/4] Fix batched silhouette score reduction aliasing --- cpp/src/stats/detail/batched/silhouette_score.cuh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cpp/src/stats/detail/batched/silhouette_score.cuh b/cpp/src/stats/detail/batched/silhouette_score.cuh index bb8a75dbf9..f41b843d5c 100644 --- a/cpp/src/stats/detail/batched/silhouette_score.cuh +++ b/cpp/src/stats/detail/batched/silhouette_score.cuh @@ -247,12 +247,15 @@ value_t silhouette_score( raft::resource::sync_stream_pool(handle); - // calculating row-wise minimum in b + // Keep the row-wise reduction output separate from b. The input is an + // n_rows x n_labels matrix, so writing an n_rows vector at b_ptr aliases + // matrix elements that may still be read by the reduction. + rmm::device_uvector b_min(n_rows, stream); raft::linalg::reduce( handle, raft::make_device_matrix_view( b_ptr, n_rows, n_labels), - raft::make_device_vector_view(b_ptr, n_rows), + raft::make_device_vector_view(b_min.data(), n_rows), std::numeric_limits::max(), false, raft::identity_op(), @@ -265,7 +268,7 @@ value_t silhouette_score( cuvs::stats::detail::SilOp(), raft::make_const_mdspan(raft::make_device_vector_view(a_ptr, n_rows)), raft::make_const_mdspan( - raft::make_device_vector_view(b_ptr, n_rows))); + raft::make_device_vector_view(b_min.data(), n_rows))); auto sum = raft::make_device_vector(handle, 1); raft::linalg::reduce( From 6960ddc880cb939851738ffddea118f67a447884 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 6 Aug 2026 13:07:31 -0500 Subject: [PATCH 2/4] Add batched silhouette score regression coverage --- cpp/tests/stats/silhouette_score.cu | 71 +++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/cpp/tests/stats/silhouette_score.cu b/cpp/tests/stats/silhouette_score.cu index 2f0d35450c..57a3565fe3 100644 --- a/cpp/tests/stats/silhouette_score.cu +++ b/cpp/tests/stats/silhouette_score.cu @@ -16,10 +16,12 @@ #include #include +#include #include #include #include #include +#include namespace cuvs { namespace stats { @@ -224,40 +226,69 @@ TEST_P(silhouetteScoreTestClass, Result) } INSTANTIATE_TEST_CASE_P(silhouetteScore, silhouetteScoreTestClass, ::testing::ValuesIn(inputs)); -TEST(silhouetteScore, BatchedStreamPoolOrdering) +TEST(silhouetteScore, BatchedMatchesNonBatchedAcrossConfigurations) { - constexpr int64_t n_rows = 4096; - constexpr int64_t n_cols = 2; - constexpr int n_labels = 2; - + constexpr int64_t n_rows = 1000; + constexpr int64_t n_cols = 2; + constexpr int n_labels = 2; + constexpr float tolerance = 1e-4f; + constexpr std::array chunks{n_rows, n_rows / 3, n_rows / 5}; + constexpr std::array metrics{cuvs::distance::DistanceType::CosineExpanded, + cuvs::distance::DistanceType::L2SqrtUnexpanded, + cuvs::distance::DistanceType::L2Expanded, + cuvs::distance::DistanceType::L1}; + + std::mt19937 rng(193); + std::uniform_real_distribution centers(-1.0f, 1.0f); + std::normal_distribution noise(0.0f, 1.5f); + std::array, n_labels> center{}; + for (auto& c : center) { + for (auto& x : c) { + x = centers(rng); + } + } + std::vector order(n_rows); + for (int64_t i = 0; i < n_rows; ++i) { + order[i] = i; + } + std::shuffle(order.begin(), order.end(), rng); std::vector X(n_rows * n_cols); std::vector labels(n_rows); - for (int64_t i = 0; i < n_rows; ++i) { - X[2 * i] = std::sin(0.01f * i); - X[2 * i + 1] = std::cos(0.013f * i); - labels[i] = i % n_labels; + for (int64_t row = 0; row < n_rows; ++row) { + auto label = static_cast(order[row] / (n_rows / n_labels)); + labels[row] = label; + for (int64_t col = 0; col < n_cols; ++col) { + X[row * n_cols + col] = center[label][col] + noise(rng); + } } - raft::resources handle; - raft::resource::set_cuda_stream_pool(handle, std::make_shared(4)); - auto stream = raft::resource::get_cuda_stream(handle); + raft::resources default_handle; + raft::resources pool_handle; + raft::resource::set_cuda_stream_pool(pool_handle, std::make_shared(4)); + auto stream = raft::resource::get_cuda_stream(default_handle); rmm::device_uvector d_X(X.size(), stream); rmm::device_uvector d_labels(labels.size(), stream); raft::update_device(d_X.data(), X.data(), X.size(), stream); raft::update_device(d_labels.data(), labels.data(), labels.size(), stream); + raft::resource::sync_stream(default_handle); auto X_view = raft::make_device_matrix_view(d_X.data(), n_rows, n_cols); auto labels_view = raft::make_device_vector_view(d_labels.data(), n_rows); - constexpr auto metric = cuvs::distance::DistanceType::L2SqrtUnexpanded; - - auto expected = - cuvs::stats::silhouette_score(handle, X_view, labels_view, std::nullopt, n_labels, metric); - for (int repeat = 0; repeat < 8; ++repeat) { - auto actual = cuvs::stats::silhouette_score_batched( - handle, X_view, labels_view, std::nullopt, n_labels, n_rows, metric); - ASSERT_NEAR(actual, expected, 1e-4f); + for (auto metric : metrics) { + auto expected = cuvs::stats::silhouette_score( + default_handle, X_view, labels_view, std::nullopt, n_labels, metric); + for (auto const& handle : + {std::pair{"default", &default_handle}, std::pair{"pool", &pool_handle}}) { + for (auto chunk : chunks) { + SCOPED_TRACE(::testing::Message() << "handle=" << handle.first << " metric=" + << static_cast(metric) << " chunk=" << chunk); + auto actual = cuvs::stats::silhouette_score_batched( + *handle.second, X_view, labels_view, std::nullopt, n_labels, chunk, metric); + ASSERT_NEAR(actual, expected, tolerance); + } + } } } From ef922834e85110882d0ba26a69c750f6662cd557 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Thu, 6 Aug 2026 13:07:41 -0500 Subject: [PATCH 3/4] Add temporary batched silhouette concurrency stress test --- cpp/tests/CMakeLists.txt | 11 + .../stats/silhouette_score_concurrency.cu | 210 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 cpp/tests/stats/silhouette_score_concurrency.cu diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 437396b736..8a4b8b5aa5 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -426,6 +426,17 @@ ConfigureTest( PERCENT 100 ) +# TEMPORARY PR DEBUGGING: BEGIN Runs the A100/CUDA-12 multi-process batched silhouette diagnostic on +# every GPU test job. Remove this registration together with stats/silhouette_score_concurrency.cu +# before merge. +ConfigureTest( + NAME BATCHED_SILHOUETTE_CONCURRENCY_TEST + PATH stats/silhouette_score_concurrency.cu + GPUS 1 + PERCENT 100 RUN_SERIAL +) +# TEMPORARY PR DEBUGGING: END + # ################################################################################################## # Install tests #################################################################################### # ################################################################################################## diff --git a/cpp/tests/stats/silhouette_score_concurrency.cu b/cpp/tests/stats/silhouette_score_concurrency.cu new file mode 100644 index 0000000000..b7056c5e01 --- /dev/null +++ b/cpp/tests/stats/silhouette_score_concurrency.cu @@ -0,0 +1,210 @@ +/* + * TEMPORARY PR DEBUGGING + * + * This executable stress-tests batched silhouette score under concurrent processes in the + * A100/CUDA-12 CI environment. Remove this entire file before merge. + * + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct metric_case { + std::string_view name; + cuvs::distance::DistanceType metric; +}; + +constexpr std::array metrics{{ + {"cosine", cuvs::distance::DistanceType::CosineExpanded}, + {"euclidean", cuvs::distance::DistanceType::L2SqrtUnexpanded}, + {"sqeuclidean", cuvs::distance::DistanceType::L2Expanded}, + {"l1", cuvs::distance::DistanceType::L1}, +}}; + +// TEMPORARY PR DEBUGGING: Worker mode preserves the diagnostic gist workload exactly while allowing +// the parent process to create the same eight-process GPU contention within one CTest executable. +int run_worker(unsigned long seed) +{ + constexpr int64_t rows = 1000; + constexpr int64_t cols = 2; + constexpr int labels = 2; + constexpr int repetitions = 4; + constexpr float tolerance = 1e-4f; + constexpr std::array chunks{rows, rows / 3, rows / 5}; + + std::mt19937 rng(seed); + std::uniform_real_distribution centers(-1.0f, 1.0f); + std::normal_distribution noise(0.0f, 1.5f); + std::array, labels> center{}; + for (auto& c : center) { + for (auto& x : c) { + x = centers(rng); + } + } + std::vector order(rows); + for (int64_t i = 0; i < rows; ++i) { + order[i] = i; + } + std::shuffle(order.begin(), order.end(), rng); + std::vector X(rows * cols); + std::vector y(rows); + for (int64_t row = 0; row < rows; ++row) { + auto label = static_cast(order[row] / (rows / labels)); + y[row] = label; + for (int64_t col = 0; col < cols; ++col) { + X[row * cols + col] = center[label][col] + noise(rng); + } + } + + raft::resources default_handle; + raft::resources pool_handle; + raft::resource::set_cuda_stream_pool(pool_handle, std::make_shared(4)); + auto stream = raft::resource::get_cuda_stream(default_handle); + auto d_X = raft::make_device_matrix(default_handle, rows, cols); + auto d_y = raft::make_device_vector(default_handle, rows); + raft::update_device(d_X.data_handle(), X.data(), X.size(), stream); + raft::update_device(d_y.data_handle(), y.data(), y.size(), stream); + raft::resource::sync_stream(default_handle); + auto X_view = raft::make_device_matrix_view(d_X.data_handle(), rows, cols); + auto y_view = raft::make_device_vector_view(d_y.data_handle(), rows); + + bool failed = false; + for (auto const& metric : metrics) { + auto non_batched = cuvs::stats::silhouette_score( + default_handle, X_view, y_view, std::nullopt, labels, metric.metric); + for (auto const& handle : + {std::pair{"default", &default_handle}, std::pair{"pool", &pool_handle}}) { + for (auto chunk : chunks) { + for (int repetition = 0; repetition < repetitions; ++repetition) { + auto batched = cuvs::stats::silhouette_score_batched( + *handle.second, X_view, y_view, std::nullopt, labels, chunk, metric.metric); + if (std::abs(batched - non_batched) > tolerance) { + failed = true; + std::cerr << std::setprecision(10) << "seed=" << seed << " handle=" << handle.first + << " metric=" << metric.name << " chunk=" << chunk + << " repetition=" << repetition << " non-batched=" << non_batched + << " batched=" << batched << " difference=" << std::abs(batched - non_batched) + << '\n'; + } + } + } + } + } + return failed ? EXIT_FAILURE : EXIT_SUCCESS; +} + +std::string executable_path() +{ + std::array path{}; + auto length = readlink("/proc/self/exe", path.data(), path.size() - 1); + if (length < 0) { + std::cerr << "readlink(/proc/self/exe) failed: " << std::strerror(errno) << '\n'; + return {}; + } + return std::string(path.data(), static_cast(length)); +} + +// TEMPORARY PR DEBUGGING: Launch seeds 0-511 in batches of eight self-exec workers to reproduce +// the process-level concurrency from the original A100/CUDA-12 diagnostic command. +int run_orchestrator() +{ + constexpr int seed_count = 512; + constexpr int process_count = 8; + auto executable = executable_path(); + if (executable.empty()) { return EXIT_FAILURE; } + + bool failed = false; + for (int first_seed = 0; first_seed < seed_count; first_seed += process_count) { + std::array children{}; + for (int offset = 0; offset < process_count; ++offset) { + auto seed = first_seed + offset; + children[offset] = fork(); + if (children[offset] == 0) { + auto seed_string = std::to_string(seed); + execl(executable.c_str(), + executable.c_str(), + "--worker", + seed_string.c_str(), + static_cast(nullptr)); + std::cerr << "seed=" << seed << " exec failed: " << std::strerror(errno) << '\n'; + _exit(127); + } + if (children[offset] < 0) { + failed = true; + std::cerr << "seed=" << seed << " fork failed: " << std::strerror(errno) << '\n'; + } + } + + for (int offset = 0; offset < process_count; ++offset) { + if (children[offset] < 0) { continue; } + int status = 0; + auto waited = waitpid(children[offset], &status, 0); + auto seed = first_seed + offset; + if (waited < 0) { + failed = true; + std::cerr << "seed=" << seed << " waitpid failed: " << std::strerror(errno) << '\n'; + } else if (WIFSIGNALED(status)) { + failed = true; + std::cerr << "seed=" << seed << " terminated by signal " << WTERMSIG(status) << '\n'; + } else if (!WIFEXITED(status) || WEXITSTATUS(status) != EXIT_SUCCESS) { + failed = true; + std::cerr << "seed=" << seed + << " worker exit code=" << (WIFEXITED(status) ? WEXITSTATUS(status) : -1) << '\n'; + } + } + } + return failed ? EXIT_FAILURE : EXIT_SUCCESS; +} + +} // namespace + +int main(int argc, char** argv) +{ + // TEMPORARY PR DEBUGGING: This private flag is used only by the self-exec stress harness. + if (argc == 3 && std::string_view(argv[1]) == "--worker") { + char* end = nullptr; + errno = 0; + auto seed = std::strtoul(argv[2], &end, 10); + if (errno != 0 || end == argv[2] || *end != '\0' || seed > 511) { + std::cerr << "invalid worker seed: " << argv[2] << '\n'; + return EXIT_FAILURE; + } + return run_worker(seed); + } + if (argc != 1) { + std::cerr << "usage: " << argv[0] << " [--worker SEED]\n"; + return EXIT_FAILURE; + } + return run_orchestrator(); +} From 20bd6694ec82b258749259df0d646df9675f63d9 Mon Sep 17 00:00:00 2001 From: Simon Adorf Date: Mon, 10 Aug 2026 15:43:07 +0000 Subject: [PATCH 4/4] Revert "Add temporary batched silhouette concurrency stress test" This reverts commit ef922834e85110882d0ba26a69c750f6662cd557. --- cpp/tests/CMakeLists.txt | 11 - .../stats/silhouette_score_concurrency.cu | 210 ------------------ 2 files changed, 221 deletions(-) delete mode 100644 cpp/tests/stats/silhouette_score_concurrency.cu diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 8a4b8b5aa5..437396b736 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -426,17 +426,6 @@ ConfigureTest( PERCENT 100 ) -# TEMPORARY PR DEBUGGING: BEGIN Runs the A100/CUDA-12 multi-process batched silhouette diagnostic on -# every GPU test job. Remove this registration together with stats/silhouette_score_concurrency.cu -# before merge. -ConfigureTest( - NAME BATCHED_SILHOUETTE_CONCURRENCY_TEST - PATH stats/silhouette_score_concurrency.cu - GPUS 1 - PERCENT 100 RUN_SERIAL -) -# TEMPORARY PR DEBUGGING: END - # ################################################################################################## # Install tests #################################################################################### # ################################################################################################## diff --git a/cpp/tests/stats/silhouette_score_concurrency.cu b/cpp/tests/stats/silhouette_score_concurrency.cu deleted file mode 100644 index b7056c5e01..0000000000 --- a/cpp/tests/stats/silhouette_score_concurrency.cu +++ /dev/null @@ -1,210 +0,0 @@ -/* - * TEMPORARY PR DEBUGGING - * - * This executable stress-tests batched silhouette score under concurrent processes in the - * A100/CUDA-12 CI environment. Remove this entire file before merge. - * - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -struct metric_case { - std::string_view name; - cuvs::distance::DistanceType metric; -}; - -constexpr std::array metrics{{ - {"cosine", cuvs::distance::DistanceType::CosineExpanded}, - {"euclidean", cuvs::distance::DistanceType::L2SqrtUnexpanded}, - {"sqeuclidean", cuvs::distance::DistanceType::L2Expanded}, - {"l1", cuvs::distance::DistanceType::L1}, -}}; - -// TEMPORARY PR DEBUGGING: Worker mode preserves the diagnostic gist workload exactly while allowing -// the parent process to create the same eight-process GPU contention within one CTest executable. -int run_worker(unsigned long seed) -{ - constexpr int64_t rows = 1000; - constexpr int64_t cols = 2; - constexpr int labels = 2; - constexpr int repetitions = 4; - constexpr float tolerance = 1e-4f; - constexpr std::array chunks{rows, rows / 3, rows / 5}; - - std::mt19937 rng(seed); - std::uniform_real_distribution centers(-1.0f, 1.0f); - std::normal_distribution noise(0.0f, 1.5f); - std::array, labels> center{}; - for (auto& c : center) { - for (auto& x : c) { - x = centers(rng); - } - } - std::vector order(rows); - for (int64_t i = 0; i < rows; ++i) { - order[i] = i; - } - std::shuffle(order.begin(), order.end(), rng); - std::vector X(rows * cols); - std::vector y(rows); - for (int64_t row = 0; row < rows; ++row) { - auto label = static_cast(order[row] / (rows / labels)); - y[row] = label; - for (int64_t col = 0; col < cols; ++col) { - X[row * cols + col] = center[label][col] + noise(rng); - } - } - - raft::resources default_handle; - raft::resources pool_handle; - raft::resource::set_cuda_stream_pool(pool_handle, std::make_shared(4)); - auto stream = raft::resource::get_cuda_stream(default_handle); - auto d_X = raft::make_device_matrix(default_handle, rows, cols); - auto d_y = raft::make_device_vector(default_handle, rows); - raft::update_device(d_X.data_handle(), X.data(), X.size(), stream); - raft::update_device(d_y.data_handle(), y.data(), y.size(), stream); - raft::resource::sync_stream(default_handle); - auto X_view = raft::make_device_matrix_view(d_X.data_handle(), rows, cols); - auto y_view = raft::make_device_vector_view(d_y.data_handle(), rows); - - bool failed = false; - for (auto const& metric : metrics) { - auto non_batched = cuvs::stats::silhouette_score( - default_handle, X_view, y_view, std::nullopt, labels, metric.metric); - for (auto const& handle : - {std::pair{"default", &default_handle}, std::pair{"pool", &pool_handle}}) { - for (auto chunk : chunks) { - for (int repetition = 0; repetition < repetitions; ++repetition) { - auto batched = cuvs::stats::silhouette_score_batched( - *handle.second, X_view, y_view, std::nullopt, labels, chunk, metric.metric); - if (std::abs(batched - non_batched) > tolerance) { - failed = true; - std::cerr << std::setprecision(10) << "seed=" << seed << " handle=" << handle.first - << " metric=" << metric.name << " chunk=" << chunk - << " repetition=" << repetition << " non-batched=" << non_batched - << " batched=" << batched << " difference=" << std::abs(batched - non_batched) - << '\n'; - } - } - } - } - } - return failed ? EXIT_FAILURE : EXIT_SUCCESS; -} - -std::string executable_path() -{ - std::array path{}; - auto length = readlink("/proc/self/exe", path.data(), path.size() - 1); - if (length < 0) { - std::cerr << "readlink(/proc/self/exe) failed: " << std::strerror(errno) << '\n'; - return {}; - } - return std::string(path.data(), static_cast(length)); -} - -// TEMPORARY PR DEBUGGING: Launch seeds 0-511 in batches of eight self-exec workers to reproduce -// the process-level concurrency from the original A100/CUDA-12 diagnostic command. -int run_orchestrator() -{ - constexpr int seed_count = 512; - constexpr int process_count = 8; - auto executable = executable_path(); - if (executable.empty()) { return EXIT_FAILURE; } - - bool failed = false; - for (int first_seed = 0; first_seed < seed_count; first_seed += process_count) { - std::array children{}; - for (int offset = 0; offset < process_count; ++offset) { - auto seed = first_seed + offset; - children[offset] = fork(); - if (children[offset] == 0) { - auto seed_string = std::to_string(seed); - execl(executable.c_str(), - executable.c_str(), - "--worker", - seed_string.c_str(), - static_cast(nullptr)); - std::cerr << "seed=" << seed << " exec failed: " << std::strerror(errno) << '\n'; - _exit(127); - } - if (children[offset] < 0) { - failed = true; - std::cerr << "seed=" << seed << " fork failed: " << std::strerror(errno) << '\n'; - } - } - - for (int offset = 0; offset < process_count; ++offset) { - if (children[offset] < 0) { continue; } - int status = 0; - auto waited = waitpid(children[offset], &status, 0); - auto seed = first_seed + offset; - if (waited < 0) { - failed = true; - std::cerr << "seed=" << seed << " waitpid failed: " << std::strerror(errno) << '\n'; - } else if (WIFSIGNALED(status)) { - failed = true; - std::cerr << "seed=" << seed << " terminated by signal " << WTERMSIG(status) << '\n'; - } else if (!WIFEXITED(status) || WEXITSTATUS(status) != EXIT_SUCCESS) { - failed = true; - std::cerr << "seed=" << seed - << " worker exit code=" << (WIFEXITED(status) ? WEXITSTATUS(status) : -1) << '\n'; - } - } - } - return failed ? EXIT_FAILURE : EXIT_SUCCESS; -} - -} // namespace - -int main(int argc, char** argv) -{ - // TEMPORARY PR DEBUGGING: This private flag is used only by the self-exec stress harness. - if (argc == 3 && std::string_view(argv[1]) == "--worker") { - char* end = nullptr; - errno = 0; - auto seed = std::strtoul(argv[2], &end, 10); - if (errno != 0 || end == argv[2] || *end != '\0' || seed > 511) { - std::cerr << "invalid worker seed: " << argv[2] << '\n'; - return EXIT_FAILURE; - } - return run_worker(seed); - } - if (argc != 1) { - std::cerr << "usage: " << argv[0] << " [--worker SEED]\n"; - return EXIT_FAILURE; - } - return run_orchestrator(); -}