From 23aed4e4bcda76de0dee037c23360eed88e43324 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Tue, 14 Jul 2026 00:53:16 -0700 Subject: [PATCH 01/24] chore(cagra): drop local dev artifacts (.clangd, .gitignore entries) --- .gitignore | 7 +++++- cpp/.clangd | 65 ----------------------------------------------------- 2 files changed, 6 insertions(+), 66 deletions(-) delete mode 100644 cpp/.clangd diff --git a/.gitignore b/.gitignore index 3627558ff5..0066d2b89a 100644 --- a/.gitignore +++ b/.gitignore @@ -72,7 +72,9 @@ docs/source/_static/rust # clang tooling compile_commands.json -.clangd/ + + + # serialized ann indexes brute_force_index @@ -86,5 +88,8 @@ ivf_pq_index /datasets/ /*.json +# clangd +*/.clangd + # java .classpath diff --git a/cpp/.clangd b/cpp/.clangd deleted file mode 100644 index 7c4fe036dd..0000000000 --- a/cpp/.clangd +++ /dev/null @@ -1,65 +0,0 @@ -# https://clangd.llvm.org/config - -# Apply a config conditionally to all C files -If: - PathMatch: .*\.(c|h)$ - ---- - -# Apply a config conditionally to all C++ files -If: - PathMatch: .*\.(c|h)pp - ---- - -# Apply a config conditionally to all CUDA files -If: - PathMatch: .*\.cuh? -CompileFlags: - Add: - - "-x" - - "cuda" - # No error on unknown CUDA versions - - "-Wno-unknown-cuda-version" - # Allow variadic CUDA functions - - "-Xclang=-fcuda-allow-variadic-functions" -Diagnostics: - Suppress: - - "variadic_device_fn" - - "attributes_not_allowed" - ---- - -# Tweak the clangd parse settings for all files -CompileFlags: - Add: - # report all errors - - "-ferror-limit=0" - - "-fmacro-backtrace-limit=0" - - "-ftemplate-backtrace-limit=0" - # Skip the CUDA version check - - "--no-cuda-version-check" - Remove: - # remove gcc's -fcoroutines - - -fcoroutines - # remove nvc++ flags unknown to clang - - "-gpu=*" - - "-stdpar*" - # remove nvcc flags unknown to clang - - "-arch*" - - "-gencode*" - - "--generate-code*" - - "-ccbin*" - - "-t=*" - - "--threads*" - - "-Xptxas*" - - "-Xcudafe*" - - "-Xfatbin*" - - "-Xcompiler*" - - "--diag-suppress*" - - "--diag_suppress*" - - "--compiler-options*" - - "--expt-extended-lambda" - - "--expt-relaxed-constexpr" - - "-forward-unknown-to-host-compiler" - - "-Werror=cross-execution-space-call" From a4ebee645418ff566f972ddfd0811147b78905be Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Tue, 14 Jul 2026 00:53:16 -0700 Subject: [PATCH 02/24] feat(cagra): add batched_device_view_from_host utility and unit test --- cpp/src/neighbors/detail/cagra/utils.hpp | 411 +++++++++++++++++- cpp/tests/CMakeLists.txt | 5 +- .../test_batched_device_view_from_host.cu | 205 +++++++++ 3 files changed, 619 insertions(+), 2 deletions(-) create mode 100644 cpp/tests/neighbors/ann_cagra/test_batched_device_view_from_host.cu diff --git a/cpp/src/neighbors/detail/cagra/utils.hpp b/cpp/src/neighbors/detail/cagra/utils.hpp index 58bf68bb43..2313631372 100644 --- a/cpp/src/neighbors/detail/cagra/utils.hpp +++ b/cpp/src/neighbors/detail/cagra/utils.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -161,6 +161,22 @@ struct gen_index_msb_1_mask { }; } // namespace utils +template +bool is_ptr_device_accessible(T* ptr) +{ + cudaPointerAttributes attr; + RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr, ptr)); + return attr.devicePointer != nullptr; +} + +template +bool is_ptr_host_accessible(T* ptr) +{ + cudaPointerAttributes attr; + RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr, ptr)); + return attr.hostPointer != nullptr; +} + /** * Utility to sync memory from a host_matrix_view to a device_matrix_view * @@ -301,4 +317,397 @@ void copy_with_padding( } } +/** + * Utility to create a batched device view from a host view + * + * This utility will create a batched device view from a host view and will handle the prefetch and + * writeback of the data Each batch can be referenced exactlyonce by calling the next_view() + * function + * + * Usage: + * ``` + * batched_device_view_from_host view(res, host_view, batch_size, host_writeback, + * initialize); while (view.next_view().extent(0) > 0) { auto device_view = view.next_view(); + * // use device_view + * } + * ``` + * + * The call to next_view() will + * * synchronize on all previous operations / increments batch_id_ + * * (optionally) write back the data of the previous batch to the host + * * (optionally) prefetch the data of the next batch + * * return the view of the current batch + * + * @tparam T The type of the data + * @tparam IdxT The type of the index + */ +template +class batched_device_view_from_host { + public: + enum class memory_strategy { + device_only, // data is on device only (no copy needed) + copy_device, // data is explicitly moved to/from device buffers + managed_only, // data is on managed memory (system managed) + }; + + /** + * Create a batched device view from a host view and will handle the prefetch and + * writeback of the data. Each batch can be referenced exactly once by calling the next_view() + * method. + * + * @param res The resources to use + * @param host_view The host view to create the batched device view from + * @param batch_size The batch size + * @param host_writeback Whether to write back the data to the host (only for host memory) + * (default: false) + * @param initialize Whether to initialize the data (only for managed memory) (default: true) + */ + batched_device_view_from_host(raft::resources const& res, + raft::host_matrix_view host_view, + uint64_t batch_size, + bool host_writeback = false, + bool initialize = true) + : res_(res), + host_view_(host_view), + batch_size_(batch_size), + offset_(0), + batch_id_(-2), + num_buffers_(2), + host_writeback_(host_writeback), + initialize_(initialize) + { + if (host_view.extent(0) == 0) { + mem_strategy_ = memory_strategy::device_only; + return; + } + + RAFT_EXPECTS(host_writeback_ || initialize_, + "At least one of host_writeback or initialize must be true"); + + RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr_, host_view.data_handle())); + switch (attr_.type) { + case cudaMemoryTypeUnregistered: + case cudaMemoryTypeHost: + case cudaMemoryTypeManaged: mem_strategy_ = memory_strategy::copy_device; break; + case cudaMemoryTypeDevice: mem_strategy_ = memory_strategy::device_only; break; + } + + RAFT_LOG_DEBUG("Memory strategy: %d for type %d, size %zu", + static_cast(mem_strategy_), + static_cast(attr_.type), + host_view.extent(0) * host_view.extent(1) * sizeof(T)); + + // buffer allocations + if (mem_strategy_ == memory_strategy::copy_device) { + try { + device_mem_[0].emplace(raft::make_device_mdarray( + res, + raft::resource::get_workspace_resource_ref(res), + raft::make_extents(batch_size, host_view.extent(1)))); + device_ptr[0] = device_mem_[0]->data_handle(); + if (batch_size < static_cast(host_view.extent(0))) { + device_mem_[1].emplace(raft::make_device_mdarray( + res, + raft::resource::get_workspace_resource_ref(res), + raft::make_extents(batch_size, host_view.extent(1)))); + device_ptr[1] = device_mem_[1]->data_handle(); + } + if (host_writeback_ && initialize_ && + batch_size * 2 < static_cast(host_view.extent(0))) { + num_buffers_ = 3; + device_mem_[2].emplace(raft::make_device_mdarray( + res, + raft::resource::get_workspace_resource_ref(res), + raft::make_extents(batch_size, host_view.extent(1)))); + device_ptr[2] = device_mem_[2]->data_handle(); + } + } catch (std::bad_alloc& e) { + if (attr_.devicePointer != nullptr) { + RAFT_LOG_DEBUG("Insufficient memory for device buffers, switching to managed memory"); + mem_strategy_ = memory_strategy::managed_only; + } else { + throw std::bad_alloc(); + } + } catch (raft::logic_error& e) { + if (attr_.devicePointer != nullptr) { + RAFT_LOG_DEBUG( + "Insufficient memory for device buffers (logic error), switching to managed memory"); + mem_strategy_ = memory_strategy::managed_only; + } else { + throw raft::logic_error("Insufficient memory for device buffers (logic error)"); + } + } + } + + // setup stream pool if not already present + size_t required_streams = host_writeback_ && initialize_ ? 2 : 1; + if (!res.has_resource_factory(raft::resource::resource_type::CUDA_STREAM_POOL) || + raft::resource::get_stream_pool_size(res) < required_streams) { + // always create at least 2 streams to account for subsequent iterator calls. + // set_cuda_stream_pool now requires a non-const resource; the referenced resource + // outlives this object, so attaching the pool to it here is safe. + raft::resource::set_cuda_stream_pool(const_cast(res), + std::make_shared(2)); + } + prefetch_stream_ = raft::resource::get_stream_from_stream_pool(res); + writeback_stream_ = raft::resource::get_stream_from_stream_pool(res); + + // if data is managed and not for_write_ we can set the attribute on the device ptr + if (mem_strategy_ == memory_strategy::managed_only) { + location_.type = cudaMemLocationTypeDevice; + location_.id = static_cast(raft::resource::get_device_id(res_)); + if (!host_writeback_) { + advise_read_mostly(host_view_.data_handle(), + host_view_.extent(0) * host_view_.extent(1) * sizeof(T)); + // TODO maybe also reset upon destruction + } + } + + // prefetch next batch (0) + prefetch_next_batch(); + } + + ~batched_device_view_from_host() noexcept + { + raft::resource::sync_stream(res_); + + // if data is on host and for_write --> make sure to copy back last active + // if data is managed and evict --> evict last active + + // make sure to sync on prefetch stream & res + switch (mem_strategy_) { + case memory_strategy::managed_only: + if (!host_writeback_) { + uint32_t discard_pos = batch_id_ % num_buffers_; + size_t discard_size_rows = actual_batch_size_[discard_pos]; + if (batch_id_ > 0) { + discard_pos = (batch_id_ - 1) % num_buffers_; + discard_size_rows += batch_size_; + } + discard_managed_region(device_ptr[discard_pos], + discard_size_rows * host_view_.extent(1) * sizeof(T)); + writeback_stream_.synchronize(); + } + break; + case memory_strategy::copy_device: + if (host_writeback_) { + uint32_t writeback_pos_last = batch_id_ % num_buffers_; + if (batch_id_ > 0) { + uint32_t writeback_pos = (batch_id_ - 1) % num_buffers_; + uint64_t writeback_offset = (batch_id_ - 1) * batch_size_; + writeback_from_device_to_host(device_ptr[writeback_pos], writeback_offset, batch_size_); + } + { + uint64_t writeback_offset_last = batch_id_ * batch_size_; + writeback_from_device_to_host(device_ptr[writeback_pos_last], + writeback_offset_last, + actual_batch_size_[writeback_pos_last]); + } + writeback_stream_.synchronize(); + } + break; + case memory_strategy::device_only: break; + } + } + + /** + * Returns the next view of the batch + * + * This function will ensure the next batch is ready and will trigger the prefetch of the + * subsequent next batch. If writeback is enabled, the last active batch will be written back to + * the host. + * + * @return The next view of the batch + */ + raft::device_matrix_view next_view() + { + bool end_of_data = static_cast((batch_id_ + 1) * batch_size_) >= + static_cast(host_view_.extent(0)); + + // special case for empty host view or last batch surpassed + if (end_of_data) { + return raft::make_device_matrix_view(nullptr, 0, host_view_.extent(1)); + } + + // trigger prefetch of next batch (also increments batch_id_) + prefetch_next_batch(); + + uint32_t current_pos = batch_id_ % num_buffers_; + return raft::make_device_matrix_view( + device_ptr[current_pos], actual_batch_size_[current_pos], host_view_.extent(1)); + } + + private: + /** + * Prefetch the next batch + * + * This function will prefetch the next batch and will handle the writeback of the data. + * + * @return True if the next batch exists, false otherwise + */ + bool prefetch_next_batch() + { + batch_id_++; + + // ensure previous batch at position batch_id_ is ready + if (initialize_) { prefetch_stream_.synchronize(); } + if (host_writeback_) { writeback_stream_.synchronize(); } + + // this step will + // * write back data from batch_id_ - 1 + // * prefetch data for batch_id_ + 1 + + // if data is on host and host_writeback_ is true we will have to copy it back + // if data is on host and initialize_ is true we will have to copy it to the device_ptr + + // if data is managed and !host_writeback_ we can discard the data from device memory + // if data is managed and initialize_ is true we can prefetch it to the device + // if data is managed and !initialize_ we can discard and prefetch the data location + + // if data is on device only this is almost a noop, just prepping the pointers + + RAFT_EXPECTS(static_cast(offset_) <= host_view_.extent(0), "Offset out of bounds"); + + bool next_batch_exists = offset_ < static_cast(host_view_.extent(0)); + + if (next_batch_exists) { + // synchronize to ensure all previous operations are completed + // in particular all work on batch_id_ - 1 + raft::resource::sync_stream(res_); + + int32_t prefetch_pos = (batch_id_ + 1) % num_buffers_; + actual_batch_size_[prefetch_pos] = min(batch_size_, host_view_.extent(0) - offset_); + + switch (mem_strategy_) { + case memory_strategy::managed_only: + if (!host_writeback_ && batch_id_ > 1) { + uint32_t discard_pos = (batch_id_ - 1) % num_buffers_; + size_t discard_size = batch_size_ * host_view_.extent(1) * sizeof(T); + discard_managed_region(device_ptr[discard_pos], discard_size); + } + // prefetch next position + device_ptr[prefetch_pos] = host_view_.data_handle() + offset_ * host_view_.extent(1); + prefetch_managed_region( + device_ptr[prefetch_pos], + actual_batch_size_[prefetch_pos] * host_view_.extent(1) * sizeof(T)); + break; + case memory_strategy::copy_device: + if (host_writeback_ && batch_id_ > 0) { + // copy back last active + uint32_t writeback_pos = (batch_id_ - 1) % num_buffers_; + uint64_t writeback_offset = (batch_id_ - 1) * batch_size_; + writeback_from_device_to_host(device_ptr[writeback_pos], writeback_offset, batch_size_); + } + if (initialize_) { + // prefetch next position + prefetch_from_host_to_device( + device_ptr[prefetch_pos], offset_, actual_batch_size_[prefetch_pos]); + } + + break; + case memory_strategy::device_only: + // just move pointer to next position + device_ptr[prefetch_pos] = host_view_.data_handle() + offset_ * host_view_.extent(1); + break; + } + + offset_ += actual_batch_size_[prefetch_pos]; + } + + return next_batch_exists; + } + + void advise_read_mostly(T* ptr, size_t size) + { +#if CUDA_VERSION >= 13000 + RAFT_CUDA_TRY(cudaMemAdvise(ptr, size, cudaMemAdviseSetReadMostly, location_)); +#else + RAFT_CUDA_TRY(cudaMemAdvise_v2(ptr, size, cudaMemAdviseSetReadMostly, location_)); +#endif + } + + void discard_managed_region(T* dev_ptr, size_t size) + { +#if CUDA_VERSION >= 13000 + void* dptrs[1] = {dev_ptr}; + size_t sizes[1] = {size}; + RAFT_CUDA_TRY(cudaMemDiscardBatchAsync(dptrs, sizes, 1, 0, writeback_stream_)); +#endif + // FIXME: CUDA12 does not support discard + } + + void prefetch_managed_region(T* dev_ptr, size_t size) + { +#if CUDA_VERSION >= 13000 + if (initialize_) { + RAFT_CUDA_TRY(cudaMemPrefetchAsync(dev_ptr, size, location_, 0, prefetch_stream_)); + } else { + void* dptrs[1] = {dev_ptr}; + size_t sizes[1] = {size}; + RAFT_CUDA_TRY( + cudaMemDiscardAndPrefetchBatchAsync(dptrs, sizes, 1, location_, 0, prefetch_stream_)); + } +#else + // FIXME: CUDA12 does not support discard - so we just prefetch + if (initialize_) { + RAFT_CUDA_TRY(cudaMemPrefetchAsync_v2(dev_ptr, size, location_, 0, prefetch_stream_)); + } else { + RAFT_CUDA_TRY(cudaMemPrefetchAsync_v2(dev_ptr, size, location_, 0, prefetch_stream_)); + } +#endif + } + + void prefetch_from_host_to_device(T* dev_ptr, size_t src_row_offset, size_t num_rows) + { + const size_t n_elem = num_rows * host_view_.extent(1); + const size_t n_bytes = n_elem * sizeof(T); + // use memcpy instead of raft::copy to avoid strange behavior with HMM/ATS memory + RAFT_CUDA_TRY(cudaMemcpyAsync(dev_ptr, + host_view_.data_handle() + src_row_offset * host_view_.extent(1), + n_bytes, + cudaMemcpyHostToDevice, + prefetch_stream_)); + } + + void writeback_from_device_to_host(T* dev_ptr, size_t dst_row_offset, size_t num_rows) + { + const size_t n_elem = num_rows * host_view_.extent(1); + const size_t n_bytes = n_elem * sizeof(T); + // use memcpy instead of raft::copy to avoid strange behavior with HMM/ATS memory + RAFT_CUDA_TRY(cudaMemcpyAsync(host_view_.data_handle() + dst_row_offset * host_view_.extent(1), + dev_ptr, + n_bytes, + cudaMemcpyDeviceToHost, + writeback_stream_)); + } + + // stream pool for local streams + std::optional> local_stream_pool_; + rmm::cuda_stream_view prefetch_stream_; + rmm::cuda_stream_view writeback_stream_; + + // configuration + memory_strategy mem_strategy_; + const raft::resources& res_; + bool initialize_; // initialize the data on the device + bool host_writeback_; // write back the data to the host + + // batch position information + uint64_t batch_size_; + int32_t batch_id_; + uint64_t offset_; + + cudaMemLocation location_; + + // input pointer information + raft::host_matrix_view host_view_; + cudaPointerAttributes attr_; + + // internal device buffers + uint64_t num_buffers_; + std::optional> device_mem_[3]; + T* device_ptr[3]; + uint32_t actual_batch_size_[3]; +}; + } // namespace cuvs::neighbors::cagra::detail diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 13a07b10b5..5e6e3b6ea6 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -183,6 +183,7 @@ ConfigureTest( neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu neighbors/ann_cagra/bug_iterative_cagra_build.cu neighbors/ann_cagra/bug_issue_93_reproducer.cu + neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu GPUS 1 PERCENT 100 ) @@ -203,7 +204,9 @@ ConfigureTest( ConfigureTest( NAME NEIGHBORS_ANN_CAGRA_HELPERS_TEST - PATH neighbors/ann_cagra/test_optimize_uint32_t.cu neighbors/ann_cagra/test_batch_load_iterator.cu + PATH neighbors/ann_cagra/test_optimize_uint32_t.cu + neighbors/ann_cagra/test_batched_device_view_from_host.cu + neighbors/ann_cagra/test_batch_load_iterator.cu GPUS 1 PERCENT 100 ) diff --git a/cpp/tests/neighbors/ann_cagra/test_batched_device_view_from_host.cu b/cpp/tests/neighbors/ann_cagra/test_batched_device_view_from_host.cu new file mode 100644 index 0000000000..1e1cc13093 --- /dev/null +++ b/cpp/tests/neighbors/ann_cagra/test_batched_device_view_from_host.cu @@ -0,0 +1,205 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "../../../src/neighbors/detail/cagra/utils.hpp" + +#include +#include +#include +#include + +namespace cuvs::neighbors::cagra { + +using IdxT = uint32_t; + +struct BatchConfig { + bool initialize; + bool host_writeback; +}; + +struct DimsConfig { + int64_t n_rows; + int64_t n_cols; + uint64_t batch_size; +}; + +class BatchedDeviceViewFromHostTest : public ::testing::Test { + protected: + void SetUp() override { raft::resource::sync_stream(res); } + + /** + * Run batched_device_view_from_host over host data, copy device views back, + * and verify against the input. + */ + template + void run_and_verify_batched(InputMatrixView input_view, + uint64_t batch_size, + bool host_writeback, + bool initialize) + { + int64_t n_rows = input_view.extent(0); + int64_t n_cols = input_view.extent(1); + + std::vector readback(n_rows * n_cols); + + int64_t total_processed = 0; + + { + cagra::detail::batched_device_view_from_host batched( + res, + raft::make_host_matrix_view(input_view.data_handle(), n_rows, n_cols), + batch_size, + host_writeback, + initialize); + while (true) { + auto dev_view = batched.next_view(); + if (dev_view.extent(0) == 0) break; + + if (initialize) { + raft::copy(readback.data() + total_processed * n_cols, + dev_view.data_handle(), + dev_view.extent(0) * dev_view.extent(1), + raft::resource::get_cuda_stream(res)); + } + if (host_writeback) { raft::matrix::fill(res, dev_view, IdxT(17)); } + total_processed += dev_view.extent(0); + } + } + raft::resource::sync_stream(res); + + EXPECT_EQ(total_processed, n_rows); + if (initialize) { + for (int64_t i = 0; i < n_rows * n_cols; ++i) { + EXPECT_EQ(readback[i], IdxT(13)) << "Mismatch (initialize) at index " << i; + } + } + if (host_writeback) { + auto readback_view = + raft::make_host_matrix_view(readback.data(), n_rows, n_cols); + raft::copy(res, readback_view, input_view); + raft::resource::sync_stream(res); + for (int64_t i = 0; i < n_rows * n_cols; ++i) { + EXPECT_EQ(readback[i], IdxT(17)) << "Mismatch (host_writeback) at index " << i; + } + } + } + + raft::resources res; +}; + +TEST_F(BatchedDeviceViewFromHostTest, EmptyView) +{ + auto host_empty = raft::make_host_matrix(0, 8); + auto host_view = host_empty.view(); + cagra::detail::batched_device_view_from_host batched( + res, host_view, /*batch_size=*/128, /*host_writeback=*/false, /*initialize=*/true); + + auto view = batched.next_view(); + EXPECT_EQ(view.extent(0), 0); + EXPECT_EQ(view.extent(1), 8); + EXPECT_EQ(view.data_handle(), nullptr); +} + +using BatchDimsParam = std::tuple; + +class BatchedDeviceViewFromHostParameterizedTest + : public BatchedDeviceViewFromHostTest, + public ::testing::WithParamInterface {}; + +TEST_P(BatchedDeviceViewFromHostParameterizedTest, VectorHostData) +{ + auto [batch_config, dims_config] = GetParam(); + auto [initialize, host_writeback] = batch_config; + auto [n_rows, n_cols, batch_size] = dims_config; + + std::vector host_data(n_rows * n_cols); + auto host_view = raft::make_host_matrix_view(host_data.data(), n_rows, n_cols); + + std::fill(host_view.data_handle(), host_view.data_handle() + n_rows * n_cols, IdxT(13)); + + run_and_verify_batched(host_view, batch_size, host_writeback, initialize); +} + +TEST_P(BatchedDeviceViewFromHostParameterizedTest, PinnedMemory) +{ + auto [batch_config, dims_config] = GetParam(); + auto [initialize, host_writeback] = batch_config; + auto [n_rows, n_cols, batch_size] = dims_config; + + auto host_matrix = raft::make_pinned_matrix(res, n_rows, n_cols); + auto host_view = host_matrix.view(); + + std::fill(host_view.data_handle(), host_view.data_handle() + n_rows * n_cols, IdxT(13)); + + run_and_verify_batched(host_view, batch_size, host_writeback, initialize); +} + +TEST_P(BatchedDeviceViewFromHostParameterizedTest, ManagedMemory) +{ + auto [batch_config, dims_config] = GetParam(); + auto [initialize, host_writeback] = batch_config; + auto [n_rows, n_cols, batch_size] = dims_config; + + auto host_matrix = raft::make_managed_matrix(res, n_rows, n_cols); + auto host_view = host_matrix.view(); + + std::fill(host_view.data_handle(), host_view.data_handle() + n_rows * n_cols, IdxT(13)); + + run_and_verify_batched(host_view, batch_size, host_writeback, initialize); +} + +TEST_P(BatchedDeviceViewFromHostParameterizedTest, DeviceMemory) +{ + auto [batch_config, dims_config] = GetParam(); + auto [initialize, host_writeback] = batch_config; + auto [n_rows, n_cols, batch_size] = dims_config; + + auto host_matrix = raft::make_device_matrix(res, n_rows, n_cols); + auto host_view = host_matrix.view(); + + raft::matrix::fill(res, host_view, IdxT(13)); + + run_and_verify_batched(host_view, batch_size, host_writeback, initialize); +} + +static const std::array kBatchConfigs = {{ + {/*initialize=*/true, /*host_writeback=*/false}, + {/*initialize=*/false, /*host_writeback=*/true}, + {/*initialize=*/true, /*host_writeback=*/true}, +}}; + +static const std::array kDimsConfigs = {{ + {/*n_rows=*/64, /*n_cols=*/32, /*batch_size=*/256}, // rows less than batch size, single batch + {/*n_rows=*/64, /*n_cols=*/32, /*batch_size=*/64}, // single batch + {/*n_rows=*/256, /*n_cols=*/32, /*batch_size=*/32}, // multiple batches + {/*n_rows=*/500, + /*n_cols=*/32, + /*batch_size=*/128}, // multiple batches, partial batch in the end +}}; + +INSTANTIATE_TEST_SUITE_P(BatchConfigs, + BatchedDeviceViewFromHostParameterizedTest, + ::testing::Combine(::testing::ValuesIn(kBatchConfigs), + ::testing::ValuesIn(kDimsConfigs))); + +} // namespace cuvs::neighbors::cagra From 90fcf80c2af29a720bc9ff18e46faed36e828169 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Tue, 14 Jul 2026 00:53:16 -0700 Subject: [PATCH 03/24] fix(cagra): bound random seed selection to graph size during build --- cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh b/cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh index e09ef82a39..f1c7305833 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh +++ b/cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh @@ -511,7 +511,11 @@ struct search hashmap.data(), hash_bitlen, stream, - static_cast(this->dataset_size)); + // Bound random seed selection to the graph size, not the dataset size. + // During iterative / CAGRA-Q build the graph is smaller than the dataset, + // so using dataset_size here selects seeds that index past the graph end + // (out-of-bounds access). See https://github.com/rapidsai/cuvs/pull/1780. + static_cast(graph.extent(0))); std::shared_ptr compute_distance_to_child_nodes_launcher = make_cagra_multi_kernel_jit_launcher Date: Tue, 14 Jul 2026 00:53:16 -0700 Subject: [PATCH 04/24] feat(cagra): iterative CAGRA-Q graph build with configurable in-build search - Configurable growth-phase in-build search params (itopk_size, search_width, max_iterations) and internal/smem dtype; itopk auto-forced on the final full-size iteration. - Decouple compression params used during iterative construction from the target index compression. - Add shuffle_dataset option; fix out-of-bounds access from the in-place raft gather by switching to an out-of-place gather. --- cpp/include/cuvs/neighbors/cagra.hpp | 276 ++++++---- cpp/include/cuvs/neighbors/common.hpp | 12 + .../neighbors/detail/cagra/cagra_build.cuh | 480 +++++++++++++++--- .../neighbors/detail/cagra/cagra_search.cuh | 4 + .../detail/cagra/compute_distance.hpp | 2 + 5 files changed, 603 insertions(+), 171 deletions(-) diff --git a/cpp/include/cuvs/neighbors/cagra.hpp b/cpp/include/cuvs/neighbors/cagra.hpp index d1937cba27..d2dabcd4d7 100644 --- a/cpp/include/cuvs/neighbors/cagra.hpp +++ b/cpp/include/cuvs/neighbors/cagra.hpp @@ -32,10 +32,172 @@ #include #include +namespace CUVS_EXPORT cuvs { +namespace neighbors { +namespace cagra { + +/** + * @defgroup cagra_cpp_search_params CAGRA index search parameters + * @{ + */ + +enum class search_algo { + /** For large batch sizes. */ + SINGLE_CTA = 0, + /** For small batch sizes. */ + MULTI_CTA = 1, + MULTI_KERNEL = 2, + AUTO = 100 +}; + +enum class hash_mode { HASH = 0, SMALL = 1, AUTO = 100 }; + +enum class internal_dtype { F16 = 0, E5M2 = 1 }; + +struct search_params : cuvs::neighbors::search_params { + /** Maximum number of queries to search at the same time (batch size). Auto select when 0.*/ + size_t max_queries = 0; + + /** Number of intermediate search results retained during the search. + * + * This is the main knob to adjust trade off between accuracy and search speed. + * Higher values improve the search accuracy. + */ + size_t itopk_size = 64; + + /** Upper limit of search iterations. Auto select when 0.*/ + size_t max_iterations = 0; + + // In the following we list additional search parameters for fine tuning. + // Reasonable default values are automatically chosen. + + /** Which search implementation to use. */ + search_algo algo = search_algo::AUTO; + + /** Number of threads used to calculate a single distance. 4, 8, 16, or 32. */ + size_t team_size = 0; + + /** Number of graph nodes to select as the starting point for the search in each iteration. aka + * search width?*/ + size_t search_width = 1; + /** Lower limit of search iterations. */ + size_t min_iterations = 0; + + /** Thread block size. 0, 64, 128, 256, 512, 1024. Auto selection when 0. */ + size_t thread_block_size = 0; + /** Hashmap type. Auto selection when AUTO. */ + hash_mode hashmap_mode = hash_mode::AUTO; + /** Lower limit of hashmap bit length. More than 8. */ + size_t hashmap_min_bitlen = 0; + /** Upper limit of hashmap fill rate. More than 0.1, less than 0.9.*/ + float hashmap_max_fill_rate = 0.5; + + /** Number of iterations of initial random seed node selection. 1 or more. */ + uint32_t num_random_samplings = 1; + /** Bit mask used for initial random seed node selection. */ + uint64_t rand_xor_mask = 0x128394; + + /** Whether to use the persistent version of the kernel (only SINGLE_CTA is supported a.t.m.) */ + bool persistent = false; + /** Persistent kernel: time in seconds before the kernel stops if no requests received. */ + float persistent_lifetime = 2; + /** + * Set the fraction of maximum grid size used by persistent kernel. + * Value 1.0 means the kernel grid size is maximum possible for the selected device. + * The value must be greater than 0.0 and not greater than 1.0. + * + * One may need to run other kernels alongside this persistent kernel. This parameter can + * be used to reduce the grid size of the persistent kernel to leave a few SMs idle. + * Note: running any other work on GPU alongside with the persistent kernel makes the setup + * fragile. + * - Running another kernel in another thread usually works, but no progress guaranteed + * - Any CUDA allocations block the context (this issue may be obscured by using pools) + * - Memory copies to not-pinned host memory may block the context + * + * Even when we know there are no other kernels working at the same time, setting + * kDeviceUsage to 1.0 surprisingly sometimes hurts performance. Proceed with care. + * If you suspect this is an issue, you can reduce this number to ~0.9 without a significant + * impact on the throughput. + */ + float persistent_device_usage = 1.0; + + /** + * A parameter indicating the rate of nodes to be filtered-out, when filtering is used. + * The value must be equal to or greater than 0.0 and less than 1.0. Default value is + * negative, in which case the filtering rate is automatically calculated when possible. + * For `filtering::udf_filter`, CAGRA uses `udf_filter::filtering_rate` when this value is + * negative. If both values are negative, CAGRA assumes 0.0 because a UDF's selectivity cannot be + * inferred from the source string. + */ + float filtering_rate = -1.0; + + /** Data type of the query vector and codebook table on shared memory. Currently, only VPQ + * supports FP8. **/ + internal_dtype smem_dtype = internal_dtype::F16; +}; + +/** + * @} + */ + +} // namespace cagra +} // namespace neighbors +} // namespace CUVS_EXPORT cuvs + namespace CUVS_EXPORT cuvs { namespace neighbors { namespace graph_build_params { -using iterative_search_params = cuvs::neighbors::search_params; +/** + * Parameters for the iterative CAGRA graph build algorithm. + * + * Inherits from cagra::search_params so that all search tuning knobs + * (search_width, max_iterations, itopk_size, etc.) are available for + * controlling the search-and-optimize loop during graph construction. + * The defaults are tuned for the build loop (e.g. search_width=1, + * max_iterations=8) and may differ from the regular search defaults. + * + * `build_compression` controls the VPQ parameters applied to the dataset + * *while building the graph*. This is independent of `index_params::compression`, + * which controls the compression of the dataset stored in the final index. + */ +struct iterative_search_params : cuvs::neighbors::cagra::search_params { + /** + * Optional VPQ compression parameters used during iterative graph construction. + * + * When set, the dataset is compressed with these parameters for the + * search-and-optimize loop. When std::nullopt (default), the builder + * falls back to `index_params::compression` (original behaviour). + */ + std::optional build_compression = std::nullopt; + + /** + * Whether to shuffle the dataset before building the graph. + * + * When enabled, the compressed dataset is randomly permuted before graph + * construction begins. This can improve graph quality by breaking any + * spatial locality in the original dataset ordering that might cause + * the iterative builder to get stuck in local optima during early + * iterations. + * + * After graph construction, the node indices in the graph are remapped + * back to the original dataset ordering. + * + * Only applies when compression is enabled (build_compression or + * index_params::compression is set). + */ + bool shuffle_dataset = true; + + iterative_search_params() + { + this->search_width = 1; + this->max_iterations = 8; + // itopk_size controls the search during the *growing* iterations of the build loop. + // 0 (default) means auto-select per iteration (max(graph_degree + 32, 128)); a nonzero + // value overrides it for the growing iterations. The final iteration always uses a fixed + // itopk tied to the output topk, regardless of this value. + this->itopk_size = 0; + } +}; /** Specialized parameters for ACE (Augmented Core Extraction) graph build */ struct ace_params { @@ -192,6 +354,14 @@ struct index_params : cuvs::neighbors::index_params { */ bool guarantee_connectivity = false; + /** + * Whether to skip graph optimization (pruning, reverse edges, MST) during non-final iterations + * of iterative graph building. When true, search results are copied directly into the device + * graph without host round-trips. Only applies to iterative_search_params graph builds; the + * final iteration always runs full optimization. + */ + bool skip_graph_optimization = false; + /** * Whether to add the dataset content to the index, i.e.: * @@ -257,110 +427,6 @@ struct index_params : cuvs::neighbors::index_params { cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded); }; -/** - * @} - */ - -/** - * @defgroup cagra_cpp_search_params CAGRA index search parameters - * @{ - */ - -enum class search_algo { - /** For large batch sizes. */ - SINGLE_CTA = 0, - /** For small batch sizes. */ - MULTI_CTA = 1, - MULTI_KERNEL = 2, - AUTO = 100 -}; - -enum class hash_mode { HASH = 0, SMALL = 1, AUTO = 100 }; - -enum class internal_dtype { F16 = 0, E5M2 = 1 }; - -struct search_params : cuvs::neighbors::search_params { - /** Maximum number of queries to search at the same time (batch size). Auto select when 0.*/ - size_t max_queries = 0; - - /** Number of intermediate search results retained during the search. - * - * This is the main knob to adjust trade off between accuracy and search speed. - * Higher values improve the search accuracy. - */ - size_t itopk_size = 64; - - /** Upper limit of search iterations. Auto select when 0.*/ - size_t max_iterations = 0; - - // In the following we list additional search parameters for fine tuning. - // Reasonable default values are automatically chosen. - - /** Which search implementation to use. */ - search_algo algo = search_algo::AUTO; - - /** Number of threads used to calculate a single distance. 4, 8, 16, or 32. */ - size_t team_size = 0; - - /** Number of graph nodes to select as the starting point for the search in each iteration. aka - * search width?*/ - size_t search_width = 1; - /** Lower limit of search iterations. */ - size_t min_iterations = 0; - - /** Thread block size. 0, 64, 128, 256, 512, 1024. Auto selection when 0. */ - size_t thread_block_size = 0; - /** Hashmap type. Auto selection when AUTO. */ - hash_mode hashmap_mode = hash_mode::AUTO; - /** Lower limit of hashmap bit length. More than 8. */ - size_t hashmap_min_bitlen = 0; - /** Upper limit of hashmap fill rate. More than 0.1, less than 0.9.*/ - float hashmap_max_fill_rate = 0.5; - - /** Number of iterations of initial random seed node selection. 1 or more. */ - uint32_t num_random_samplings = 1; - /** Bit mask used for initial random seed node selection. */ - uint64_t rand_xor_mask = 0x128394; - - /** Whether to use the persistent version of the kernel (only SINGLE_CTA is supported a.t.m.) */ - bool persistent = false; - /** Persistent kernel: time in seconds before the kernel stops if no requests received. */ - float persistent_lifetime = 2; - /** - * Set the fraction of maximum grid size used by persistent kernel. - * Value 1.0 means the kernel grid size is maximum possible for the selected device. - * The value must be greater than 0.0 and not greater than 1.0. - * - * One may need to run other kernels alongside this persistent kernel. This parameter can - * be used to reduce the grid size of the persistent kernel to leave a few SMs idle. - * Note: running any other work on GPU alongside with the persistent kernel makes the setup - * fragile. - * - Running another kernel in another thread usually works, but no progress guaranteed - * - Any CUDA allocations block the context (this issue may be obscured by using pools) - * - Memory copies to not-pinned host memory may block the context - * - * Even when we know there are no other kernels working at the same time, setting - * kDeviceUsage to 1.0 surprisingly sometimes hurts performance. Proceed with care. - * If you suspect this is an issue, you can reduce this number to ~0.9 without a significant - * impact on the throughput. - */ - float persistent_device_usage = 1.0; - - /** - * A parameter indicating the rate of nodes to be filtered-out, when filtering is used. - * The value must be equal to or greater than 0.0 and less than 1.0. Default value is - * negative, in which case the filtering rate is automatically calculated when possible. - * For `filtering::udf_filter`, CAGRA uses `udf_filter::filtering_rate` when this value is - * negative. If both values are negative, CAGRA assumes 0.0 because a UDF's selectivity cannot be - * inferred from the source string. - */ - float filtering_rate = -1.0; - - /** Data type of the query vector and codebook table on shared memory. Currently, only VPQ - * supports FP8. **/ - internal_dtype smem_dtype = internal_dtype::F16; -}; - /** * @} */ diff --git a/cpp/include/cuvs/neighbors/common.hpp b/cpp/include/cuvs/neighbors/common.hpp index 2fd804f115..1e5ca5a159 100644 --- a/cpp/include/cuvs/neighbors/common.hpp +++ b/cpp/include/cuvs/neighbors/common.hpp @@ -98,6 +98,18 @@ struct vpq_params { * The max number of data points to use per VQ cluster during training. */ uint32_t max_train_points_per_vq_cluster = 1024; + + friend bool operator==(const vpq_params& a, const vpq_params& b) + { + return a.pq_bits == b.pq_bits && a.pq_dim == b.pq_dim && a.vq_n_centers == b.vq_n_centers && + a.kmeans_n_iters == b.kmeans_n_iters && + a.vq_kmeans_trainset_fraction == b.vq_kmeans_trainset_fraction && + a.pq_kmeans_trainset_fraction == b.pq_kmeans_trainset_fraction && + a.pq_kmeans_type == b.pq_kmeans_type && + a.max_train_points_per_pq_code == b.max_train_points_per_pq_code && + a.max_train_points_per_vq_cluster == b.max_train_points_per_vq_cluster; + } + friend bool operator!=(const vpq_params& a, const vpq_params& b) { return !(a == b); } }; /** @} */ // end group cagra_cpp_index_params diff --git a/cpp/src/neighbors/detail/cagra/cagra_build.cuh b/cpp/src/neighbors/detail/cagra/cagra_build.cuh index 774254c84c..f0547beecb 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_build.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_build.cuh @@ -19,8 +19,17 @@ #include #include #include +#include +#include +#include +#include #include +#include +#include +#include +#include + #include #include #include @@ -53,6 +62,32 @@ namespace cuvs::neighbors::cagra::detail { constexpr double to_mib(size_t bytes) { return static_cast(bytes) / (1 << 20); } constexpr double to_gib(size_t bytes) { return static_cast(bytes) / (1 << 30); } +// Functor to remap indices using a permutation lookup table +template +struct remap_indices_op { + const IdxT* perm; + __host__ __device__ IdxT operator()(IdxT idx) const { return perm[idx]; } +}; + +// Functor to compute scattered output index for graph row reordering +template +struct graph_scatter_index_op { + const IdxT* perm; + int64_t degree; + __host__ __device__ int64_t operator()(int64_t idx) const + { + int64_t row = idx / degree; + int64_t col = idx % degree; + return static_cast(perm[row]) * degree + col; + } +}; + +// Functor to convert int64_t to IdxT +template +struct cast_to_idx_op { + __host__ __device__ IdxT operator()(int64_t v) const { return static_cast(v); } +}; + template void check_graph_degree(size_t& intermediate_degree, size_t& graph_degree, size_t dataset_size) { @@ -2016,6 +2051,110 @@ struct mmap_owner { size_t size_; }; +template +__global__ void kern_reconstruct_vpq_queries(const uint8_t* encoded_data, + uint32_t encoded_row_len, + const MathT* vq_codebook, + const MathT* pq_codebook, + uint32_t dim, + uint32_t pq_len, + uint64_t offset, + uint32_t batch_size, + T* output) +{ + const uint64_t batch_idx = blockIdx.x; + if (batch_idx >= batch_size) return; + const uint64_t vec_idx = offset + batch_idx; + const uint8_t* vec_data = encoded_data + vec_idx * encoded_row_len; + const uint32_t vq_code = *reinterpret_cast(vec_data); + const uint8_t* pq_codes = vec_data + sizeof(uint32_t); + const MathT* vq_centroid_ptr = vq_codebook + static_cast(vq_code) * dim; + + for (uint32_t d = threadIdx.x; d < dim; d += blockDim.x) { + uint32_t j = d / pq_len; + uint32_t k = d % pq_len; + float val = static_cast(vq_centroid_ptr[d]) + + static_cast(pq_codebook[static_cast(pq_codes[j]) * pq_len + k]); + output[batch_idx * dim + d] = static_cast(val); + } +} + +template +void reconstruct_vpq_queries(raft::resources const& res, + const vpq_dataset& vpq_dset, + uint64_t offset, + uint32_t batch_size, + raft::device_matrix_view output) +{ + const uint32_t dim = vpq_dset.dim(); + const uint32_t pq_len = vpq_dset.pq_len(); + const uint32_t threads = std::min(dim, 256u); + + kern_reconstruct_vpq_queries + <<>>( + vpq_dset.data.data_handle(), + vpq_dset.encoded_row_length(), + vpq_dset.vq_code_book.data_handle(), + vpq_dset.pq_code_book.data_handle(), + dim, + pq_len, + offset, + batch_size, + output.data_handle()); +} + +template +void search_and_optimize(raft::resources const& res, + const cuvs::neighbors::cagra::search_params& search_params, + const index& idx, + raft::device_matrix_view dev_query_view, + raft::device_matrix_view dev_neighbors, + raft::device_matrix_view dev_distances, + raft::device_matrix& dev_output_graph, + size_t curr_query_size, + size_t next_graph_degree, + size_t curr_topk, + uint64_t max_chunk_size) +{ + auto stream = raft::resource::get_cuda_stream(res); + + auto dev_knn_graph = raft::make_device_matrix(res, curr_query_size, curr_topk); + + auto query_batch = cuvs::spatial::knn::detail::utils::make_batch_load_iterator( + res, + dev_query_view.data_handle(), + static_cast(curr_query_size), + static_cast(dev_query_view.extent(1)), + max_chunk_size, + stream, + raft::resource::get_workspace_resource_ref(res)); + for (const auto& batch : query_batch) { + auto batch_dev_query_view = raft::make_device_matrix_view( + batch.data(), batch.size(), dev_query_view.extent(1)); + auto batch_dev_neighbors_view = raft::make_device_matrix_view( + dev_neighbors.data_handle(), batch.size(), curr_topk); + auto batch_dev_distances_view = raft::make_device_matrix_view( + dev_distances.data_handle(), batch.size(), curr_topk); + + cuvs::neighbors::cagra::search(res, + search_params, + idx, + batch_dev_query_view, + batch_dev_neighbors_view, + batch_dev_distances_view); + + raft::copy(dev_knn_graph.data_handle() + batch.offset() * curr_topk, + batch_dev_neighbors_view.data_handle(), + batch.size() * curr_topk, + stream); + } + + dev_output_graph = + raft::make_device_matrix(res, curr_query_size, next_graph_degree); + + graph::optimize(res, dev_knn_graph.view(), dev_output_graph.view(), false); +} + template (params.graph_build_params); + const auto& build_compression = + iter_params.build_compression.has_value() ? iter_params.build_compression : params.compression; + + if (build_compression.has_value()) { + const auto& bc = *build_compression; + RAFT_LOG_INFO( + "Build compression params: pq_bits=%u, pq_dim=%u, vq_n_centers=%u, kmeans_n_iters=%u, " + "vq_kmeans_trainset_fraction=%.4f, pq_kmeans_trainset_fraction=%.4f, " + "max_train_points_per_pq_code=%u, max_train_points_per_vq_cluster=%u%s", + bc.pq_bits, + bc.pq_dim, + bc.vq_n_centers, + bc.kmeans_n_iters, + bc.vq_kmeans_trainset_fraction, + bc.pq_kmeans_trainset_fraction, + bc.max_train_points_per_pq_code, + bc.max_train_points_per_vq_cluster, + iter_params.build_compression.has_value() ? " (from build_compression)" + : " (from compression)"); + } else { + RAFT_LOG_INFO("Build compression: disabled (uncompressed build)"); + } + RAFT_LOG_INFO("Build search params: search_width=%zu, max_iterations=%zu", + iter_params.search_width, + iter_params.max_iterations); + auto cagra_graph = raft::make_host_matrix(0, 0); // Iteratively improve the accuracy of the graph by repeatedly running @@ -2078,6 +2248,17 @@ auto iterative_build_graph( RAFT_LOG_DEBUG("# graph_degree = %lu", (uint64_t)graph_degree); RAFT_LOG_DEBUG("# topk = %lu", (uint64_t)topk); + // A fixed itopk_size (0 = auto) governs the growing iterations, which build graphs of degree + // ~graph_degree/2 and thus request topk ~= graph_degree/2 + 1; the search planner requires + // topk <= itopk_size. (The full-size iterations override itopk internally, so they are not + // constrained by this value.) + RAFT_EXPECTS(iter_params.itopk_size == 0 || + iter_params.itopk_size >= graph_degree / 2 + 1, + "iterative build search itopk_size (%zu) must be 0 (auto) or >= " + "graph_degree / 2 + 1 (%zu)", + (size_t)iter_params.itopk_size, + (size_t)(graph_degree / 2 + 1)); + // Create an initial graph. The initial graph created here is not suitable for // searching, but connectivity is guaranteed. auto offset = raft::make_host_vector(small_graph_degree); @@ -2098,28 +2279,131 @@ auto iterative_build_graph( } } - // Allocate memory for neighbors list using Transparent HugePage - constexpr size_t thp_size = 2 * 1024 * 1024; - size_t byte_size = sizeof(IdxT) * final_graph_size * topk; - if (byte_size % thp_size) { byte_size += thp_size - (byte_size % thp_size); } - mmap_owner neighbors_list(byte_size); - IdxT* neighbors_ptr = (IdxT*)neighbors_list.data(); - memset(neighbors_ptr, 0, byte_size); - bool flag_last = false; auto curr_graph_size = initial_graph_size; + + auto dev_graph = raft::make_device_matrix(res, 0, 0); + bool use_device_graph = false; + + // Generate the compressed index once if compression is enabled + const uint64_t dataset_dim = dev_dataset.extent(1); + std::optional> idx_opt; + + // Optional shuffle permutation for randomizing dataset order during build. + // inverse_perm[shuffled_idx] = original_idx + // perm[shuffled_idx] = original_idx, used to unshuffle the graph after build + auto dev_perm = raft::make_device_vector(res, 0); + bool dataset_shuffled = false; + + // Warn if shuffle is requested but compression is not enabled + if (iter_params.shuffle_dataset && !build_compression.has_value()) { + RAFT_LOG_WARN("shuffle_dataset is only supported with compression enabled; ignoring"); + } + + if (build_compression.has_value()) { + auto start = std::chrono::high_resolution_clock::now(); + RAFT_EXPECTS(params.metric == cuvs::distance::DistanceType::L2Expanded, + "VPQ compression is only supported with L2Expanded distance mertric"); + + // Build the VPQ compressed dataset + auto vpq_dset = + cuvs::preprocessing::quantize::pq::vpq_build(res, *build_compression, dev_dataset); + + // Optionally shuffle the compressed dataset to break spatial locality + if (iter_params.shuffle_dataset) { + auto shuffle_start = std::chrono::high_resolution_clock::now(); + RAFT_LOG_INFO("Shuffling compressed dataset to randomize build order..."); + + auto stream = raft::resource::get_cuda_stream(res); + const auto n_rows = vpq_dset.data.extent(0); + const auto row_len = vpq_dset.data.extent(1); + + // Generate random permutation: perm[i] = source index for output row i + // i.e., shuffled_data[i] = original_data[perm[i]] + // So perm maps: shuffled_idx -> original_idx + // Use int64_t for permutation to match vpq_dataset's index type + auto dev_perm_i64 = raft::make_device_vector(res, n_rows); + + // Use legacy permute API to generate permutation indices only (out=nullptr, in=nullptr) + // This just fills dev_perm_i64 with a random permutation of [0, n_rows) + raft::random::permute(dev_perm_i64.data_handle(), + static_cast(nullptr), + static_cast(nullptr), + static_cast(row_len), + static_cast(n_rows), + true, + stream); + + // Apply permutation to VPQ data: shuffled_data[i] = original_data[perm[i]]. + // NOTE: use an out-of-place device gather into a temporary buffer rather than the + // in-place gather overload. The in-place overload uses a host-orchestrated, + // double-buffered, multi-stream path that races here and triggers an asynchronous + // illegal memory access (the crash disappears under CUDA_LAUNCH_BLOCKING=1). + auto shuffled_data = raft::make_device_matrix( + res, vpq_dset.data.extent(0), vpq_dset.data.extent(1)); + raft::matrix::gather(res, + raft::make_const_mdspan(vpq_dset.data.view()), + raft::make_const_mdspan(dev_perm_i64.view()), + shuffled_data.view()); + vpq_dset.data = std::move(shuffled_data); + + // Store perm as IdxT for graph unshuffling later + // perm[shuffled_idx] = original_idx + // This is used for: + // 1. Remapping neighbor values: neighbor j (shuffled) -> perm[j] (original) + // 2. Reordering rows: row i (for shuffled node i) -> position perm[i] (original node) + dev_perm = raft::make_device_vector(res, n_rows); + cast_to_idx_op cast_op; + thrust::transform(raft::resource::get_thrust_policy(res), + dev_perm_i64.data_handle(), + dev_perm_i64.data_handle() + n_rows, + dev_perm.data_handle(), + cast_op); + + dataset_shuffled = true; + + auto shuffle_end = std::chrono::high_resolution_clock::now(); + auto shuffle_ms = + std::chrono::duration_cast(shuffle_end - shuffle_start).count(); + RAFT_LOG_INFO("# Dataset shuffle time: %.3lf sec", (double)shuffle_ms / 1000); + } + + idx_opt.emplace(res, params.metric); + // Use the (optionally shuffled) compressed dataset built above. + idx_opt->update_dataset(res, std::move(vpq_dset)); + auto end = std::chrono::high_resolution_clock::now(); + auto elapsed_ms = std::chrono::duration_cast(end - start).count(); + RAFT_LOG_INFO("# VPQ compression time: %.3lf sec", (double)elapsed_ms / 1000); + + // Free the original dataset -- queries will be reconstructed from VPQ codes. + dev_aligned_dataset.reset(); + RAFT_LOG_INFO( + "# Freed original dataset from device (%.1f MiB); queries will use VPQ reconstruction", + to_mib(final_graph_size * dataset_dim * sizeof(T))); + } while (true) { auto start = std::chrono::high_resolution_clock::now(); auto curr_query_size = std::min(2 * curr_graph_size, final_graph_size); auto next_graph_degree = small_graph_degree; if (curr_graph_size == final_graph_size) { next_graph_degree = graph_degree; } + RAFT_LOG_INFO("Current graph size %lu: # current graph degree = %lu", + (uint64_t)curr_graph_size, + (uint64_t)next_graph_degree); // The search count (topk) is set to the next graph degree + 1, because // pruning is not used except in the last iteration. // (*) The appropriate setting for itopk_size requires careful consideration. - auto curr_topk = next_graph_degree + 1; - auto curr_itopk_size = next_graph_degree + 32; + auto curr_topk = next_graph_degree + 1; + // The configurable itopk (iter_params.itopk_size, 0 = auto) applies only to the true growing + // iterations, where the degree being built is small_graph_degree. When the graph reaches its + // full size the search builds a graph_degree-degree graph (topk = graph_degree + 1); that + // iteration needs a larger itopk, so it overrides the configured value with the auto formula. + // The final iteration (flag_last) uses a fixed itopk tied to the output topk. + auto curr_itopk_size = + (iter_params.itopk_size > 0 && next_graph_degree == small_graph_degree) + ? (uint64_t)iter_params.itopk_size + : std::max(next_graph_degree + 32, (uint64_t)128); if (flag_last) { curr_topk = topk; curr_itopk_size = curr_topk + 32; @@ -2134,71 +2418,135 @@ auto iterative_build_graph( (uint64_t)curr_itopk_size, (uint64_t)curr_topk); - cuvs::neighbors::cagra::search_params search_params; - search_params.algo = cuvs::neighbors::cagra::search_algo::AUTO; - search_params.max_queries = max_chunk_size; - search_params.itopk_size = curr_itopk_size; - - // Create an index (idx), a query view (dev_query_view), and a mdarray for - // search results (neighbors). - auto dev_dataset_view = raft::make_device_matrix_view( - dev_dataset.data_handle(), (int64_t)curr_graph_size, dev_dataset.extent(1)); - - auto idx = index( - res, params.metric, dev_dataset_view, raft::make_const_mdspan(cagra_graph.view())); - - auto dev_query_view = raft::make_device_matrix_view( - dev_dataset.data_handle(), (int64_t)curr_query_size, dev_dataset.extent(1)); - - auto neighbors_view = - raft::make_host_matrix_view(neighbors_ptr, curr_query_size, curr_topk); - - // Search. - // Since there are many queries, divide them into batches and search them. - auto query_batch = cuvs::spatial::knn::detail::utils::make_batch_load_iterator( - res, - dev_query_view.data_handle(), - static_cast(curr_query_size), - static_cast(dev_query_view.extent(1)), - max_chunk_size, - raft::resource::get_cuda_stream(res), - raft::resource::get_workspace_resource_ref(res)); - for (const auto& batch : query_batch) { - auto batch_dev_query_view = raft::make_device_matrix_view( - batch.data(), batch.size(), dev_query_view.extent(1)); - auto batch_dev_neighbors_view = raft::make_device_matrix_view( - dev_neighbors.data_handle(), batch.size(), curr_topk); - auto batch_dev_distances_view = raft::make_device_matrix_view( - dev_distances.data_handle(), batch.size(), curr_topk); - - cuvs::neighbors::cagra::search(res, - search_params, - idx, - batch_dev_query_view, - batch_dev_neighbors_view, - batch_dev_distances_view); - - auto batch_neighbors_view = raft::make_host_matrix_view( - neighbors_view.data_handle() + batch.offset() * curr_topk, batch.size(), curr_topk); - raft::copy(res, batch_neighbors_view, batch_dev_neighbors_view); + cuvs::neighbors::cagra::search_params search_params = iter_params; + search_params.max_queries = max_chunk_size; + search_params.itopk_size = curr_itopk_size; + + // Create index and query views. + if (!build_compression.has_value()) { + auto dev_dataset_view = raft::make_device_matrix_view( + dev_dataset.data_handle(), (int64_t)curr_graph_size, dev_dataset.extent(1)); + if (use_device_graph) { + idx_opt.emplace( + res, params.metric, dev_dataset_view, raft::make_const_mdspan(dev_graph.view())); + } else { + idx_opt.emplace( + res, params.metric, dev_dataset_view, raft::make_const_mdspan(cagra_graph.view())); + } + } else { + if (use_device_graph) { + idx_opt->update_graph(res, raft::make_const_mdspan(dev_graph.view())); + } else { + idx_opt->update_graph(res, raft::make_const_mdspan(cagra_graph.view())); + } } - - // Optimize graph - auto next_graph_size = curr_query_size; - cagra_graph = raft::make_host_matrix(0, 0); // delete existing grahp - cagra_graph = raft::make_host_matrix(next_graph_size, next_graph_degree); - optimize( - res, neighbors_view, cagra_graph.view(), flag_last ? params.guarantee_connectivity : 0); + const auto& idx = *idx_opt; + + // When compression is enabled, reconstruct queries from VPQ codes instead of + // reading from the (freed) original dataset. + auto dev_reconstructed_queries = + build_compression.has_value() + ? raft::make_device_matrix(res, curr_query_size, dataset_dim) + : raft::make_device_matrix(res, 0, 0); + if (build_compression.has_value()) { + auto* vpq_dset = dynamic_cast*>(&idx.data()); + RAFT_EXPECTS(vpq_dset != nullptr, "Expected VPQ dataset in compressed index"); + reconstruct_vpq_queries( + res, *vpq_dset, 0, curr_query_size, dev_reconstructed_queries.view()); + } + auto dev_query_view = + build_compression.has_value() + ? raft::make_device_matrix_view( + dev_reconstructed_queries.data_handle(), (int64_t)curr_query_size, dataset_dim) + : raft::make_device_matrix_view( + dev_dataset.data_handle(), (int64_t)curr_query_size, dev_dataset.extent(1)); + + auto dev_optimized_graph = raft::make_device_matrix(res, 0, 0); + + search_and_optimize(res, + search_params, + idx, + dev_query_view, + dev_neighbors.view(), + dev_distances.view(), + dev_optimized_graph, + curr_query_size, + next_graph_degree, + curr_topk, + max_chunk_size); + + dev_graph = std::move(dev_optimized_graph); + use_device_graph = true; auto end = std::chrono::high_resolution_clock::now(); auto elapsed_ms = std::chrono::duration_cast(end - start).count(); RAFT_LOG_DEBUG("# elapsed time: %.3lf sec", (double)elapsed_ms / 1000); if (flag_last) { break; } - flag_last = (curr_graph_size == final_graph_size); - curr_graph_size = next_graph_size; + flag_last = (curr_graph_size == final_graph_size); + auto next_graph_size = curr_query_size; + curr_graph_size = next_graph_size; } + // TODO: when build_compression matches params.compression, the dataset is compressed twice + // (once for the build loop and once in build()'s shared tail). We could avoid this by returning + // the index directly (with its VPQ dataset and device-side graph) instead of just the host graph. + auto stream = raft::resource::get_cuda_stream(res); + + // If the dataset was shuffled, we need to unshuffle the graph: + // Recall: perm[shuffled_idx] = original_idx (stored in dev_perm) + // 1. Remap neighbor indices from shuffled space to original space + // 2. Reorder rows from shuffled order to original order + if (dataset_shuffled) { + auto unshuffle_start = std::chrono::high_resolution_clock::now(); + RAFT_LOG_INFO("Unshuffling graph to restore original dataset ordering..."); + + const auto n_rows = dev_graph.extent(0); + const auto degree = dev_graph.extent(1); + + // Step 1: Remap all neighbor indices using perm + // graph[i][j] contains shuffled index j; we need original index = perm[j] + remap_indices_op remap_op{dev_perm.data_handle()}; + thrust::transform(raft::resource::get_thrust_policy(res), + dev_graph.data_handle(), + dev_graph.data_handle() + n_rows * degree, + dev_graph.data_handle(), + remap_op); + + // Step 2: Reorder rows back to original order + // Row i in dev_graph is for shuffled node i, which is original node perm[i]. + // We want this row to be at position perm[i] in the final graph. + // scatter: output[map[i]] = input[i], so map[i] = perm[i] + auto dev_unshuffled_graph = raft::make_device_matrix(res, n_rows, degree); + + // Use thrust::scatter to reorder: for each row i, place it at position perm[i] + // We scatter row-by-row conceptually, but do it element-wise with computed output indices + graph_scatter_index_op scatter_idx_op{dev_perm.data_handle(), degree}; + auto output_indices = + thrust::make_transform_iterator(thrust::make_counting_iterator(0), scatter_idx_op); + + thrust::scatter(raft::resource::get_thrust_policy(res), + dev_graph.data_handle(), + dev_graph.data_handle() + n_rows * degree, + output_indices, + dev_unshuffled_graph.data_handle()); + + dev_graph = std::move(dev_unshuffled_graph); + + auto unshuffle_end = std::chrono::high_resolution_clock::now(); + auto unshuffle_ms = + std::chrono::duration_cast(unshuffle_end - unshuffle_start) + .count(); + RAFT_LOG_INFO("# Graph unshuffle time: %.3lf sec", (double)unshuffle_ms / 1000); + } + + cagra_graph = raft::make_host_matrix(dev_graph.extent(0), dev_graph.extent(1)); + raft::copy(cagra_graph.data_handle(), + dev_graph.data_handle(), + dev_graph.extent(0) * dev_graph.extent(1), + stream); + raft::resource::sync_stream(res); + return cagra_graph; } diff --git a/cpp/src/neighbors/detail/cagra/cagra_search.cuh b/cpp/src/neighbors/detail/cagra/cagra_search.cuh index 4d09e3683b..265df89c59 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_search.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_search.cuh @@ -73,11 +73,13 @@ void search_main_core( topk, queries.extent(1)); + RAFT_LOG_DEBUG("search_main_core: creating plan with max_node_id=%u", params.max_node_id); using CagraSampleFilterT_s = typename CagraSampleFilterT_Selector::type; std::unique_ptr< search_plan_impl> plan = factory::create( res, params, dataset_desc, queries.extent(1), graph.extent(0), graph.extent(1), topk); + RAFT_LOG_DEBUG("search_main_core: plan created, plan->max_node_id=%u", plan->max_node_id); plan->check(topk); @@ -158,6 +160,7 @@ void search_main(raft::resources const& res, params.smem_dtype = cuvs::neighbors::cagra::internal_dtype::F16; } // Search using a plain (strided) row-major dataset + RAFT_LOG_DEBUG("Searching with strided dataset"); RAFT_EXPECTS(index.metric() != cuvs::distance::DistanceType::CosineExpanded || index.dataset_norms().has_value(), "Dataset norms must be provided for CosineExpanded metric"); @@ -184,6 +187,7 @@ void search_main(raft::resources const& res, RAFT_FAIL("FP32 VPQ dataset support is coming soon"); } else if (auto* vpq_dset = dynamic_cast*>(&index.data()); vpq_dset != nullptr) { + RAFT_LOG_DEBUG("Searching with VPQ dataset"); if (params.smem_dtype == cuvs::neighbors::cagra::internal_dtype::E5M2 && raft::getComputeCapability().first < 9) { RAFT_LOG_WARN( diff --git a/cpp/src/neighbors/detail/cagra/compute_distance.hpp b/cpp/src/neighbors/detail/cagra/compute_distance.hpp index a99ec64bc0..28cc6b6eba 100644 --- a/cpp/src/neighbors/detail/cagra/compute_distance.hpp +++ b/cpp/src/neighbors/detail/cagra/compute_distance.hpp @@ -229,6 +229,8 @@ struct dataset_descriptor_host { ~state() noexcept { if (std::holds_alternative(value)) { + // RAFT_LOG_INFO("trying to free descriptor state %p", + // reinterpret_cast(this)); auto& [ptr, stream] = std::get(value); RAFT_CUDA_TRY_NO_THROW(cudaFreeAsync(ptr, stream)); } From 90d121963e6d3348ce66280d5958ef2331639a41 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Tue, 14 Jul 2026 00:53:16 -0700 Subject: [PATCH 05/24] feat(bench): expose iterative CAGRA-Q build/search params in cuvs_bench --- .../src/cuvs/cuvs_ann_bench_param_parser.h | 98 ++++++++++++++++++- python/cuvs_bench/cuvs_bench/run/__main__.py | 8 ++ 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h index 57b47d97db..82db80d2e7 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h +++ b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h @@ -367,10 +367,12 @@ void parse_build_param(const nlohmann::json& conf, cuvs::neighbors::cagra::index } // Parse build-algo-specific parameters and use them to decide on the algo type - nlohmann::json ivf_pq_build_conf = collect_conf_with_prefix(conf, "ivf_pq_build_"); - nlohmann::json ivf_pq_search_conf = collect_conf_with_prefix(conf, "ivf_pq_search_"); - nlohmann::json nn_descent_conf = collect_conf_with_prefix(conf, "nn_descent_"); - nlohmann::json ace_conf = collect_conf_with_prefix(conf, "ace_"); + nlohmann::json ivf_pq_build_conf = collect_conf_with_prefix(conf, "ivf_pq_build_"); + nlohmann::json ivf_pq_search_conf = collect_conf_with_prefix(conf, "ivf_pq_search_"); + nlohmann::json nn_descent_conf = collect_conf_with_prefix(conf, "nn_descent_"); + nlohmann::json ace_conf = collect_conf_with_prefix(conf, "ace_"); + nlohmann::json build_compression_conf = collect_conf_with_prefix(conf, "build_compression_"); + nlohmann::json build_search_conf = collect_conf_with_prefix(conf, "build_search_"); // When graph_build_algo is not specified, leave graph_build_params as monostate so the // CAGRA build uses AUTO selection (NN_DESCENT or IVF_PQ based on dataset/heuristics). @@ -401,6 +403,94 @@ void parse_build_param(const nlohmann::json& conf, cuvs::neighbors::cagra::index } else if constexpr (std::is_same_v) { parse_build_param(nn_descent_conf, arg); + } else if constexpr (std::is_same_v< + U, + cuvs::neighbors::graph_build_params::iterative_search_params>) { + if (!build_compression_conf.empty()) { + auto vpq_pams = arg.build_compression.value_or(cuvs::neighbors::vpq_params{}); + parse_build_param(build_compression_conf, vpq_pams); + arg.build_compression.emplace(vpq_pams); + } + if (build_search_conf.contains("width")) { + arg.search_width = build_search_conf.at("width"); + } + if (build_search_conf.contains("max_iterations")) { + arg.max_iterations = build_search_conf.at("max_iterations"); + } + if (build_search_conf.contains("min_iterations")) { + arg.min_iterations = build_search_conf.at("min_iterations"); + } + if (build_search_conf.contains("itopk")) { arg.itopk_size = build_search_conf.at("itopk"); } + if (build_search_conf.contains("max_queries")) { + arg.max_queries = build_search_conf.at("max_queries"); + } + if (build_search_conf.contains("team_size")) { + arg.team_size = build_search_conf.at("team_size"); + } + if (build_search_conf.contains("thread_block_size")) { + arg.thread_block_size = build_search_conf.at("thread_block_size"); + } + if (build_search_conf.contains("hashmap_min_bitlen")) { + arg.hashmap_min_bitlen = build_search_conf.at("hashmap_min_bitlen"); + } + if (build_search_conf.contains("hashmap_max_fill_rate")) { + arg.hashmap_max_fill_rate = build_search_conf.at("hashmap_max_fill_rate"); + } + if (build_search_conf.contains("num_random_samplings")) { + arg.num_random_samplings = build_search_conf.at("num_random_samplings"); + } + if (build_search_conf.contains("persistent")) { + arg.persistent = build_search_conf.at("persistent"); + } + if (build_search_conf.contains("persistent_lifetime")) { + arg.persistent_lifetime = build_search_conf.at("persistent_lifetime"); + } + if (build_search_conf.contains("persistent_device_usage")) { + arg.persistent_device_usage = build_search_conf.at("persistent_device_usage"); + } + if (build_search_conf.contains("algo")) { + std::string algo = build_search_conf.at("algo"); + if (algo == "single_cta") { + arg.algo = cuvs::neighbors::cagra::search_algo::SINGLE_CTA; + } else if (algo == "multi_cta") { + arg.algo = cuvs::neighbors::cagra::search_algo::MULTI_CTA; + } else if (algo == "multi_kernel") { + arg.algo = cuvs::neighbors::cagra::search_algo::MULTI_KERNEL; + } else if (algo == "auto") { + arg.algo = cuvs::neighbors::cagra::search_algo::AUTO; + } + } + if (build_search_conf.contains("hashmap_mode")) { + std::string mode = build_search_conf.at("hashmap_mode"); + if (mode == "hash") { + arg.hashmap_mode = cuvs::neighbors::cagra::hash_mode::HASH; + } else if (mode == "small") { + arg.hashmap_mode = cuvs::neighbors::cagra::hash_mode::SMALL; + } else if (mode == "auto") { + arg.hashmap_mode = cuvs::neighbors::cagra::hash_mode::AUTO; + } + } + // Whether to shuffle the (compressed) dataset before the iterative build loop. + if (build_search_conf.contains("shuffle_dataset")) { + arg.shuffle_dataset = build_search_conf.at("shuffle_dataset").get(); + } + // Precision of the codebook/query in shared memory for the VPQ search used during + // the iterative build. Accepts an integer code (0=F16, 1=E5M2) or a string. + if (build_search_conf.contains("smem_dtype")) { + const auto& sd = build_search_conf.at("smem_dtype"); + if (sd.is_number_integer()) { + arg.smem_dtype = static_cast(sd.get()); + } else { + std::string s = sd.get(); + if (s == "f16" || s == "F16" || s == "fp16" || s == "half") { + arg.smem_dtype = cuvs::neighbors::cagra::internal_dtype::F16; + } else if (s == "e5m2" || s == "E5M2" || s == "fp8") { + arg.smem_dtype = cuvs::neighbors::cagra::internal_dtype::E5M2; + } else { + throw std::runtime_error("invalid value for build_search smem_dtype: " + s); + } + } + } } }, params.graph_build_params); diff --git a/python/cuvs_bench/cuvs_bench/run/__main__.py b/python/cuvs_bench/cuvs_bench/run/__main__.py index 6950ff7202..58d1b604bd 100644 --- a/python/cuvs_bench/cuvs_bench/run/__main__.py +++ b/python/cuvs_bench/cuvs_bench/run/__main__.py @@ -5,6 +5,7 @@ import json import os +import warnings from pathlib import Path from typing import Optional @@ -257,6 +258,13 @@ def main( and any backend-specific connection parameters (host, port, etc.). """ + warnings.warn( + "The 'cuvs_bench.run' CLI is deprecated and will be removed in a future release. " + "Use BenchmarkOrchestrator from cuvs_bench.orchestrator instead.", + FutureWarning, + stacklevel=2, + ) + if not data_export: # Determine backend type and extra kwargs from --backend-config backend_type = "cpp_gbench" From a9389588737ec7cf15688dad83b23a05db823c4e Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Tue, 14 Jul 2026 00:53:16 -0700 Subject: [PATCH 06/24] test(cagra): VPQ/iterative build test updates --- cpp/tests/neighbors/ann_cagra.cuh | 104 +++++++++--------- .../bug_graph_smaller_than_dataset.cu | 20 ++-- cpp/tests/neighbors/ann_utils.cuh | 22 ++-- 3 files changed, 73 insertions(+), 73 deletions(-) diff --git a/cpp/tests/neighbors/ann_cagra.cuh b/cpp/tests/neighbors/ann_cagra.cuh index 7b86cc70ad..1f7b5c6977 100644 --- a/cpp/tests/neighbors/ann_cagra.cuh +++ b/cpp/tests/neighbors/ann_cagra.cuh @@ -1547,38 +1547,38 @@ inline std::vector generate_inputs() {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL}); inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - // Corner cases for small datasets - inputs2 = raft::util::itertools::product( - {2}, - {3, 6, 31, 32, 64, 101}, - {1, 10}, - {2}, // k - {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - {search_algo::SINGLE_CTA, search_algo::MULTI_CTA, search_algo::MULTI_KERNEL}, - {0}, // query size - {0}, - {256}, - {1}, - {cuvs::distance::DistanceType::L2Expanded}, - {false}, - {true}, - {true}, - {0.995}, - {std::optional{std::nullopt}}, - {std::optional{std::nullopt}}, - {std::optional{std::nullopt}}, - {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL, - cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_LOGICAL}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); + // // Corner cases for small datasets + // inputs2 = raft::util::itertools::product( + // {2}, + // {3, 6, 31, 32, 64, 101}, + // {1, 10}, + // {2}, // k + // {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, + // {search_algo::SINGLE_CTA, search_algo::MULTI_CTA, search_algo::MULTI_KERNEL}, + // {0}, // query size + // {0}, + // {256}, + // {1}, + // {cuvs::distance::DistanceType::L2Expanded}, + // {false}, + // {true}, + // {true}, + // {0.995}, + // {std::optional{std::nullopt}}, + // {std::optional{std::nullopt}}, + // {std::optional{std::nullopt}}, + // {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL, + // cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_LOGICAL}); + // inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); // Varying dim and build algo. inputs2 = raft::util::itertools::product( {100}, - {1000}, - {1, 3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 1024}, // dim - {16}, // k - {graph_build_algo::IVF_PQ, - graph_build_algo::NN_DESCENT, + {1000000}, + {768}, // dim + {16}, // k + { // graph_build_algo::IVF_PQ, + // graph_build_algo::NN_DESCENT, graph_build_algo::ITERATIVE_CAGRA_SEARCH}, {search_algo::AUTO}, {10}, @@ -1592,7 +1592,7 @@ inline std::vector generate_inputs() {false}, {true}, {false}, - {0.995}, + {0.01}, {std::optional{std::nullopt}}, {std::optional{std::nullopt}}, {std::optional{std::nullopt}}, @@ -1657,29 +1657,29 @@ inline std::vector generate_inputs() {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL}); inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - // Varying n_rows, host_dataset - inputs2 = raft::util::itertools::product( - {100}, - {10000}, - {32}, - {10}, - {graph_build_algo::AUTO}, - {search_algo::AUTO}, - {10}, - {0}, // team_size - {64}, - {1}, - {cuvs::distance::DistanceType::L2Expanded, cuvs::distance::DistanceType::InnerProduct}, - {false, true}, - {false}, - {true}, - {0.985}, - {std::optional{std::nullopt}}, - {std::optional{std::nullopt}}, - {std::optional{std::nullopt}}, - {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL, - cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_LOGICAL}); - inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); + // // Varying n_rows, host_dataset + // inputs2 = raft::util::itertools::product( + // {100}, + // {10000}, + // {32}, + // {10}, + // {graph_build_algo::AUTO}, + // {search_algo::AUTO}, + // {10}, + // {0}, // team_size + // {64}, + // {1}, + // {cuvs::distance::DistanceType::L2Expanded, cuvs::distance::DistanceType::InnerProduct}, + // {false, true}, + // {false}, + // {true}, + // {0.985}, + // {std::optional{std::nullopt}}, + // {std::optional{std::nullopt}}, + // {std::optional{std::nullopt}}, + // {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL, + // cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_LOGICAL}); + // inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); // A few PQ configurations. // Varying dim, vq_n_centers diff --git a/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu b/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu index adeb774a8b..b06c1cba92 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu @@ -38,8 +38,8 @@ class cagra_graph_smaller_than_dataset_test : public ::testing::Test { protected: void run() { - // Create a dataset with 1000 points - constexpr int64_t n_dataset = 1000; + // Create a dataset with 10000 points + constexpr int64_t n_dataset = 10000; constexpr int64_t n_dim = 128; constexpr int64_t n_queries = 100; constexpr int64_t k = 10; @@ -63,9 +63,9 @@ class cagra_graph_smaller_than_dataset_test : public ::testing::Test { // Recreate the bug scenario: LARGE dataset, SMALL graph // (like iterative_build_graph does in intermediate iterations) - constexpr int64_t n_graph = n_dataset / 2; // Only 500 nodes in graph + constexpr int64_t n_graph = n_dataset / 2; // Only 5000 nodes in graph - // Step 1: Build index on SMALL subset (500 points) + // Step 1: Build index on SMALL subset (5000 points) auto small_dataset_view = raft::make_device_matrix_view( dataset.data_handle(), n_graph, n_dim); @@ -74,13 +74,13 @@ class cagra_graph_smaller_than_dataset_test : public ::testing::Test { auto small_index = cagra::build(res, small_index_params, small_dataset_view); raft::resource::sync_stream(res); - // Step 2: Update to FULL dataset (1000 points) but keep small graph (500 nodes) - // This creates the exact bug scenario: dataset.size=1000, graph.extent(0)=500 + // Step 2: Update to FULL dataset (10000 points) but keep small graph (5000 nodes) + // This creates the exact bug scenario: dataset.size=10000, graph.extent(0)=5000 small_index.update_dataset(res, raft::make_const_mdspan(dataset.view())); // Verify the mismatch - THIS IS THE BUG SCENARIO! - ASSERT_EQ(small_index.graph().extent(0), n_graph); // Graph has 500 nodes - ASSERT_EQ(small_index.size(), n_dataset); // Dataset has 1000 points + ASSERT_EQ(small_index.graph().extent(0), n_graph); // Graph has 5000 nodes + ASSERT_EQ(small_index.size(), n_dataset); // Dataset has 10000 points ASSERT_NE(small_index.graph().extent(0), small_index.size()); // Mismatch! // Create queries @@ -100,8 +100,8 @@ class cagra_graph_smaller_than_dataset_test : public ::testing::Test { search_params.algo = cagra::search_algo::SINGLE_CTA; // THIS SHOULD NOT CRASH OR CAUSE OOB ACCESS - // Before fix: random seeds use dataset.size (1000) -> tries to access graph[700] -> CRASH! - // After fix: random seeds use graph.extent(0) (500) -> only accesses graph[0-499] -> SAFE! + // Before fix: random seeds use dataset.size (10000) -> tries to access graph[7000] -> CRASH! + // After fix: random seeds use graph.extent(0) (5000) -> only accesses graph[0-4999] -> SAFE! cagra::search(res, search_params, small_index, diff --git a/cpp/tests/neighbors/ann_utils.cuh b/cpp/tests/neighbors/ann_utils.cuh index 7240363ee4..e3dcbea6c6 100644 --- a/cpp/tests/neighbors/ann_utils.cuh +++ b/cpp/tests/neighbors/ann_utils.cuh @@ -127,10 +127,10 @@ struct idx_dist_pair { /** Calculate recall value using only neighbor indices */ template -auto calc_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - size_t rows, - size_t cols) +std::tuple calc_recall(const std::vector& expected_idx, + const std::vector& actual_idx, + size_t rows, + size_t cols) { size_t match_count = 0; size_t total_count = static_cast(rows) * static_cast(cols); @@ -219,13 +219,13 @@ auto eval_recall(const std::vector& expected_idx, /** Overload of calc_recall to account for distances */ template -auto calc_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - const std::vector& expected_dist, - const std::vector& actual_dist, - size_t rows, - size_t cols, - double eps) +std::tuple calc_recall(const std::vector& expected_idx, + const std::vector& actual_idx, + const std::vector& expected_dist, + const std::vector& actual_dist, + size_t rows, + size_t cols, + double eps) { size_t match_count = 0; size_t index_match_count = 0; From 4f4068ab15be7391680161c1b1d72b47b7b8e416 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Tue, 14 Jul 2026 03:18:17 -0700 Subject: [PATCH 07/24] fix(cagra): use in-place gather for dataset shuffle (remove raft workaround) The shuffle_dataset path used an out-of-place gather into a temporary buffer to work around an illegal memory access in raft's in-place gather overload when n_rows * row_len exceeded 2^31 (32-bit index overflow). That bug is now fixed upstream in raft (rapidsai/raft#3059, closes #3055), which the cuvs raft pin now includes. Revert to the in-place gather to drop the extra full-size temporary allocation and copy. --- .../neighbors/detail/cagra/cagra_build.cuh | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/cpp/src/neighbors/detail/cagra/cagra_build.cuh b/cpp/src/neighbors/detail/cagra/cagra_build.cuh index f0547beecb..7021132a5d 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_build.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_build.cuh @@ -2252,8 +2252,7 @@ auto iterative_build_graph( // ~graph_degree/2 and thus request topk ~= graph_degree/2 + 1; the search planner requires // topk <= itopk_size. (The full-size iterations override itopk internally, so they are not // constrained by this value.) - RAFT_EXPECTS(iter_params.itopk_size == 0 || - iter_params.itopk_size >= graph_degree / 2 + 1, + RAFT_EXPECTS(iter_params.itopk_size == 0 || iter_params.itopk_size >= graph_degree / 2 + 1, "iterative build search itopk_size (%zu) must be 0 (auto) or >= " "graph_degree / 2 + 1 (%zu)", (size_t)iter_params.itopk_size, @@ -2334,18 +2333,13 @@ auto iterative_build_graph( true, stream); - // Apply permutation to VPQ data: shuffled_data[i] = original_data[perm[i]]. - // NOTE: use an out-of-place device gather into a temporary buffer rather than the - // in-place gather overload. The in-place overload uses a host-orchestrated, - // double-buffered, multi-stream path that races here and triggers an asynchronous - // illegal memory access (the crash disappears under CUDA_LAUNCH_BLOCKING=1). - auto shuffled_data = raft::make_device_matrix( - res, vpq_dset.data.extent(0), vpq_dset.data.extent(1)); - raft::matrix::gather(res, - raft::make_const_mdspan(vpq_dset.data.view()), - raft::make_const_mdspan(dev_perm_i64.view()), - shuffled_data.view()); - vpq_dset.data = std::move(shuffled_data); + // Apply permutation to VPQ data in place: data[i] = original_data[perm[i]]. + // Previously this used an out-of-place gather into a temporary buffer to work around + // an illegal memory access in the in-place gather overload when n_rows * row_len + // exceeded 2^31 (32-bit index overflow). That bug is fixed upstream in raft + // (rapidsai/raft#3059, issue #3055), which cuvs now pins, so the in-place gather is + // safe again and avoids the extra full-size temporary allocation and copy. + raft::matrix::gather(res, vpq_dset.data.view(), raft::make_const_mdspan(dev_perm_i64.view())); // Store perm as IdxT for graph unshuffling later // perm[shuffled_idx] = original_idx @@ -2400,10 +2394,9 @@ auto iterative_build_graph( // full size the search builds a graph_degree-degree graph (topk = graph_degree + 1); that // iteration needs a larger itopk, so it overrides the configured value with the auto formula. // The final iteration (flag_last) uses a fixed itopk tied to the output topk. - auto curr_itopk_size = - (iter_params.itopk_size > 0 && next_graph_degree == small_graph_degree) - ? (uint64_t)iter_params.itopk_size - : std::max(next_graph_degree + 32, (uint64_t)128); + auto curr_itopk_size = (iter_params.itopk_size > 0 && next_graph_degree == small_graph_degree) + ? (uint64_t)iter_params.itopk_size + : std::max(next_graph_degree + 32, (uint64_t)128); if (flag_last) { curr_topk = topk; curr_itopk_size = curr_topk + 32; From cc6529185191cad3ecf1bcf81bb43e89c706b002 Mon Sep 17 00:00:00 2001 From: aamijar Date: Wed, 22 Jul 2026 22:07:33 +0000 Subject: [PATCH 08/24] fix style --- cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h | 2 +- cpp/include/cuvs/neighbors/common.hpp | 2 +- cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh | 2 +- cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu | 2 +- .../neighbors/ann_cagra/test_batched_device_view_from_host.cu | 2 +- python/cuvs_bench/cuvs_bench/run/__main__.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h index 82db80d2e7..cd830cea68 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h +++ b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/neighbors/common.hpp b/cpp/include/cuvs/neighbors/common.hpp index 1e5ca5a159..f7b935a86d 100644 --- a/cpp/include/cuvs/neighbors/common.hpp +++ b/cpp/include/cuvs/neighbors/common.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh b/cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh index f1c7305833..ee834a0b24 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh +++ b/cpp/src/neighbors/detail/cagra/search_multi_kernel.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu b/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu index b06c1cba92..844471dd2d 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_cagra/test_batched_device_view_from_host.cu b/cpp/tests/neighbors/ann_cagra/test_batched_device_view_from_host.cu index 1e1cc13093..eb72dbec92 100644 --- a/cpp/tests/neighbors/ann_cagra/test_batched_device_view_from_host.cu +++ b/cpp/tests/neighbors/ann_cagra/test_batched_device_view_from_host.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/python/cuvs_bench/cuvs_bench/run/__main__.py b/python/cuvs_bench/cuvs_bench/run/__main__.py index 58d1b604bd..7f52b9c49b 100644 --- a/python/cuvs_bench/cuvs_bench/run/__main__.py +++ b/python/cuvs_bench/cuvs_bench/run/__main__.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # From 25e6d7dedd6f5dd53835d4f144cf625b431f0aa4 Mon Sep 17 00:00:00 2001 From: aamijar Date: Wed, 22 Jul 2026 22:19:06 +0000 Subject: [PATCH 09/24] restore clangd and gitignore changes --- .gitignore | 7 +----- cpp/.clangd | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 cpp/.clangd diff --git a/.gitignore b/.gitignore index 0066d2b89a..3627558ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -72,9 +72,7 @@ docs/source/_static/rust # clang tooling compile_commands.json - - - +.clangd/ # serialized ann indexes brute_force_index @@ -88,8 +86,5 @@ ivf_pq_index /datasets/ /*.json -# clangd -*/.clangd - # java .classpath diff --git a/cpp/.clangd b/cpp/.clangd new file mode 100644 index 0000000000..7c4fe036dd --- /dev/null +++ b/cpp/.clangd @@ -0,0 +1,65 @@ +# https://clangd.llvm.org/config + +# Apply a config conditionally to all C files +If: + PathMatch: .*\.(c|h)$ + +--- + +# Apply a config conditionally to all C++ files +If: + PathMatch: .*\.(c|h)pp + +--- + +# Apply a config conditionally to all CUDA files +If: + PathMatch: .*\.cuh? +CompileFlags: + Add: + - "-x" + - "cuda" + # No error on unknown CUDA versions + - "-Wno-unknown-cuda-version" + # Allow variadic CUDA functions + - "-Xclang=-fcuda-allow-variadic-functions" +Diagnostics: + Suppress: + - "variadic_device_fn" + - "attributes_not_allowed" + +--- + +# Tweak the clangd parse settings for all files +CompileFlags: + Add: + # report all errors + - "-ferror-limit=0" + - "-fmacro-backtrace-limit=0" + - "-ftemplate-backtrace-limit=0" + # Skip the CUDA version check + - "--no-cuda-version-check" + Remove: + # remove gcc's -fcoroutines + - -fcoroutines + # remove nvc++ flags unknown to clang + - "-gpu=*" + - "-stdpar*" + # remove nvcc flags unknown to clang + - "-arch*" + - "-gencode*" + - "--generate-code*" + - "-ccbin*" + - "-t=*" + - "--threads*" + - "-Xptxas*" + - "-Xcudafe*" + - "-Xfatbin*" + - "-Xcompiler*" + - "--diag-suppress*" + - "--diag_suppress*" + - "--compiler-options*" + - "--expt-extended-lambda" + - "--expt-relaxed-constexpr" + - "-forward-unknown-to-host-compiler" + - "-Werror=cross-execution-space-call" From 6bca27512894cfcf22a4cd5af5d3d6e658145c40 Mon Sep 17 00:00:00 2001 From: aamijar Date: Wed, 22 Jul 2026 22:34:03 +0000 Subject: [PATCH 10/24] revert cuvs_bench warning --- python/cuvs_bench/cuvs_bench/run/__main__.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/run/__main__.py b/python/cuvs_bench/cuvs_bench/run/__main__.py index 7f52b9c49b..29dcbd1f41 100644 --- a/python/cuvs_bench/cuvs_bench/run/__main__.py +++ b/python/cuvs_bench/cuvs_bench/run/__main__.py @@ -1,11 +1,10 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # import json import os -import warnings from pathlib import Path from typing import Optional @@ -258,12 +257,6 @@ def main( and any backend-specific connection parameters (host, port, etc.). """ - warnings.warn( - "The 'cuvs_bench.run' CLI is deprecated and will be removed in a future release. " - "Use BenchmarkOrchestrator from cuvs_bench.orchestrator instead.", - FutureWarning, - stacklevel=2, - ) if not data_export: # Determine backend type and extra kwargs from --backend-config From 6ed4ab5967737a09b9a7d2f4b6ae9e25a320a7d8 Mon Sep 17 00:00:00 2001 From: aamijar Date: Wed, 22 Jul 2026 22:35:08 +0000 Subject: [PATCH 11/24] remove whitespace --- python/cuvs_bench/cuvs_bench/run/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/cuvs_bench/cuvs_bench/run/__main__.py b/python/cuvs_bench/cuvs_bench/run/__main__.py index 29dcbd1f41..6950ff7202 100644 --- a/python/cuvs_bench/cuvs_bench/run/__main__.py +++ b/python/cuvs_bench/cuvs_bench/run/__main__.py @@ -257,7 +257,6 @@ def main( and any backend-specific connection parameters (host, port, etc.). """ - if not data_export: # Determine backend type and extra kwargs from --backend-config backend_type = "cpp_gbench" From 9d44b2374fc3417099e135021956374acf353640 Mon Sep 17 00:00:00 2001 From: aamijar Date: Wed, 22 Jul 2026 22:50:44 +0000 Subject: [PATCH 12/24] remove duplicate file in cmakelists.txt --- cpp/tests/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 46c53c4f49..e50470ecdc 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -187,7 +187,6 @@ ConfigureTest( neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu neighbors/ann_cagra/bug_iterative_cagra_build.cu neighbors/ann_cagra/bug_issue_93_reproducer.cu - neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu GPUS 1 PERCENT 100 ) From 73c9bbcdbe23cdd9993e730b91b080b9c6f34d0e Mon Sep 17 00:00:00 2001 From: aamijar Date: Wed, 22 Jul 2026 22:52:38 +0000 Subject: [PATCH 13/24] revert to auto for type deduction --- cpp/tests/neighbors/ann_utils.cuh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cpp/tests/neighbors/ann_utils.cuh b/cpp/tests/neighbors/ann_utils.cuh index e3dcbea6c6..7240363ee4 100644 --- a/cpp/tests/neighbors/ann_utils.cuh +++ b/cpp/tests/neighbors/ann_utils.cuh @@ -127,10 +127,10 @@ struct idx_dist_pair { /** Calculate recall value using only neighbor indices */ template -std::tuple calc_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - size_t rows, - size_t cols) +auto calc_recall(const std::vector& expected_idx, + const std::vector& actual_idx, + size_t rows, + size_t cols) { size_t match_count = 0; size_t total_count = static_cast(rows) * static_cast(cols); @@ -219,13 +219,13 @@ auto eval_recall(const std::vector& expected_idx, /** Overload of calc_recall to account for distances */ template -std::tuple calc_recall(const std::vector& expected_idx, - const std::vector& actual_idx, - const std::vector& expected_dist, - const std::vector& actual_dist, - size_t rows, - size_t cols, - double eps) +auto calc_recall(const std::vector& expected_idx, + const std::vector& actual_idx, + const std::vector& expected_dist, + const std::vector& actual_dist, + size_t rows, + size_t cols, + double eps) { size_t match_count = 0; size_t index_match_count = 0; From 23cf81575eca343de0fec701cf8d83ab062a638e Mon Sep 17 00:00:00 2001 From: aamijar Date: Wed, 22 Jul 2026 22:59:22 +0000 Subject: [PATCH 14/24] remove commented out code --- cpp/src/neighbors/detail/cagra/compute_distance.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/neighbors/detail/cagra/compute_distance.hpp b/cpp/src/neighbors/detail/cagra/compute_distance.hpp index 28cc6b6eba..a99ec64bc0 100644 --- a/cpp/src/neighbors/detail/cagra/compute_distance.hpp +++ b/cpp/src/neighbors/detail/cagra/compute_distance.hpp @@ -229,8 +229,6 @@ struct dataset_descriptor_host { ~state() noexcept { if (std::holds_alternative(value)) { - // RAFT_LOG_INFO("trying to free descriptor state %p", - // reinterpret_cast(this)); auto& [ptr, stream] = std::get(value); RAFT_CUDA_TRY_NO_THROW(cudaFreeAsync(ptr, stream)); } From c2f9b6a6ae7311e5923746e2a4f061154c9ad069 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Thu, 23 Jul 2026 06:49:18 -0700 Subject: [PATCH 15/24] fix(cagra): pass graph_size to persistent single-CTA kernel to bound build-time random seeds --- .../neighbors/detail/cagra/jit_lto_kernels/kernel_def.hpp | 1 + .../detail/cagra/jit_lto_kernels/search_single_cta_jit.cuh | 6 ++++-- .../cagra/jit_lto_kernels/search_single_cta_p_kernel.cu.in | 2 ++ .../detail/cagra/search_single_cta_kernel_launcher_jit.cuh | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/kernel_def.hpp b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/kernel_def.hpp index c9702d3b72..a686f29921 100644 --- a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/kernel_def.hpp +++ b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/kernel_def.hpp @@ -76,6 +76,7 @@ using search_single_cta_p_kernel_func_t = const std::uint32_t, const std::uint32_t, const dataset_descriptor_base_t*, + const IndexT, cagra_sample_filter); } // namespace single_cta_search diff --git a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_single_cta_jit.cuh b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_single_cta_jit.cuh index d481c44946..8cc39de507 100644 --- a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_single_cta_jit.cuh +++ b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_single_cta_jit.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -551,6 +551,7 @@ __device__ void search_single_cta_p_impl( const std::uint32_t small_hash_reset_interval, const std::uint32_t query_id_offset, // Offset to add to query_id when calling filter const dataset_descriptor_base_t* dataset_desc, + const IndexT graph_size, cagra_sample_filter filter_payload) { using job_desc_type = job_desc_t>; @@ -625,7 +626,8 @@ __device__ void search_single_cta_p_impl( query_id, query_id_offset, dataset_desc, - filter_payload); + filter_payload, + graph_size); // make sure all writes are visible even for the host // (e.g. when result buffers are in pinned memory) diff --git a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_single_cta_p_kernel.cu.in b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_single_cta_p_kernel.cu.in index 9986f7abc1..b003220497 100644 --- a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_single_cta_p_kernel.cu.in +++ b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_single_cta_p_kernel.cu.in @@ -51,6 +51,7 @@ extern "C" __global__ __launch_bounds__(1024, 1) void search_single_cta_p( const std::uint32_t small_hash_reset_interval, const std::uint32_t query_id_offset, const dataset_desc_base* dataset_desc, + const index_t graph_size, cagra_sample_filter_t filter_payload) { search_single_cta_p_impl(graph.extent(0)), filter_payload); last_touch.store(std::chrono::system_clock::now(), std::memory_order_relaxed); From cad2e8f5608c1c8f37e3b7f9414ec6c4cf405329 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Thu, 6 Aug 2026 00:28:41 -0700 Subject: [PATCH 16/24] Brought back the tests --- cpp/tests/neighbors/ann_cagra.cuh | 96 +++++++++++++++---------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/cpp/tests/neighbors/ann_cagra.cuh b/cpp/tests/neighbors/ann_cagra.cuh index 0419f30166..f3b45ee5b8 100644 --- a/cpp/tests/neighbors/ann_cagra.cuh +++ b/cpp/tests/neighbors/ann_cagra.cuh @@ -1598,30 +1598,30 @@ inline std::vector generate_inputs() {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL}); inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - // // Corner cases for small datasets - // inputs2 = raft::util::itertools::product( - // {2}, - // {3, 6, 31, 32, 64, 101}, - // {1, 10}, - // {2}, // k - // {32}, // degree - // {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, - // {search_algo::SINGLE_CTA, search_algo::MULTI_CTA, search_algo::MULTI_KERNEL}, - // {0}, // query size - // {0}, - // {256}, - // {1}, - // {cuvs::distance::DistanceType::L2Expanded}, - // {false}, - // {true}, - // {true}, - // {0.995}, - // {std::optional{std::nullopt}}, - // {std::optional{std::nullopt}}, - // {std::optional{std::nullopt}}, - // {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL, - // cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_LOGICAL}); - // inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); + // Corner cases for small datasets + inputs2 = raft::util::itertools::product( + {2}, + {3, 6, 31, 32, 64, 101}, + {1, 10}, + {2}, // k + {32}, // degree + {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT}, + {search_algo::SINGLE_CTA, search_algo::MULTI_CTA, search_algo::MULTI_KERNEL}, + {0}, // query size + {0}, + {256}, + {1}, + {cuvs::distance::DistanceType::L2Expanded}, + {false}, + {true}, + {true}, + {0.995}, + {std::optional{std::nullopt}}, + {std::optional{std::nullopt}}, + {std::optional{std::nullopt}}, + {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL, + cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_LOGICAL}); + inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); // Varying dim and build algo. inputs2 = raft::util::itertools::product( @@ -1712,30 +1712,30 @@ inline std::vector generate_inputs() {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL}); inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); - // // Varying n_rows, host_dataset - // inputs2 = raft::util::itertools::product( - // {100}, - // {10000}, - // {32}, - // {10}, - // {32}, // degree - // {graph_build_algo::AUTO}, - // {search_algo::AUTO}, - // {10}, - // {0}, // team_size - // {64}, - // {1}, - // {cuvs::distance::DistanceType::L2Expanded, cuvs::distance::DistanceType::InnerProduct}, - // {false, true}, - // {false}, - // {true}, - // {0.985}, - // {std::optional{std::nullopt}}, - // {std::optional{std::nullopt}}, - // {std::optional{std::nullopt}}, - // {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL, - // cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_LOGICAL}); - // inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); + // Varying n_rows, host_dataset + inputs2 = raft::util::itertools::product( + {100}, + {10000}, + {32}, + {10}, + {32}, // degree + {graph_build_algo::AUTO}, + {search_algo::AUTO}, + {10}, + {0}, // team_size + {64}, + {1}, + {cuvs::distance::DistanceType::L2Expanded, cuvs::distance::DistanceType::InnerProduct}, + {false, true}, + {false}, + {true}, + {0.985}, + {std::optional{std::nullopt}}, + {std::optional{std::nullopt}}, + {std::optional{std::nullopt}}, + {cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL, + cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_LOGICAL}); + inputs.insert(inputs.end(), inputs2.begin(), inputs2.end()); // A few PQ configurations. // Varying dim, vq_n_centers From 0e98b34efea8f771e7b0b6f1c3fddcba422d7ec2 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Thu, 6 Aug 2026 07:33:21 -0700 Subject: [PATCH 17/24] Fixed the test --- cpp/tests/neighbors/ann_cagra.cuh | 12 +++++------ .../bug_graph_smaller_than_dataset.cu | 20 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cpp/tests/neighbors/ann_cagra.cuh b/cpp/tests/neighbors/ann_cagra.cuh index f3b45ee5b8..f85e36a6ef 100644 --- a/cpp/tests/neighbors/ann_cagra.cuh +++ b/cpp/tests/neighbors/ann_cagra.cuh @@ -1626,13 +1626,13 @@ inline std::vector generate_inputs() // Varying dim and build algo. inputs2 = raft::util::itertools::product( {100}, - {1000000}, - {768}, // dim + {1000}, + {1, 3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 768, 1024}, // dim {16}, // k {32}, // degree - { // graph_build_algo::IVF_PQ, - // graph_build_algo::NN_DESCENT, - graph_build_algo::ITERATIVE_CAGRA_SEARCH}, + {graph_build_algo::IVF_PQ, + graph_build_algo::NN_DESCENT, + graph_build_algo::ITERATIVE_CAGRA_SEARCH}, // Iterative cagra q build {search_algo::AUTO}, {10}, {0}, @@ -1645,7 +1645,7 @@ inline std::vector generate_inputs() {false}, {true}, {false}, - {0.01}, + {0.995}, {std::optional{std::nullopt}}, {std::optional{std::nullopt}}, {std::optional{std::nullopt}}, diff --git a/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu b/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu index 844471dd2d..16fa93f47b 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu @@ -38,8 +38,8 @@ class cagra_graph_smaller_than_dataset_test : public ::testing::Test { protected: void run() { - // Create a dataset with 10000 points - constexpr int64_t n_dataset = 10000; + // Create a dataset with 1000 points + constexpr int64_t n_dataset = 1000; constexpr int64_t n_dim = 128; constexpr int64_t n_queries = 100; constexpr int64_t k = 10; @@ -63,9 +63,9 @@ class cagra_graph_smaller_than_dataset_test : public ::testing::Test { // Recreate the bug scenario: LARGE dataset, SMALL graph // (like iterative_build_graph does in intermediate iterations) - constexpr int64_t n_graph = n_dataset / 2; // Only 5000 nodes in graph + constexpr int64_t n_graph = n_dataset / 2; // Only 500 nodes in graph - // Step 1: Build index on SMALL subset (5000 points) + // Step 1: Build index on SMALL subset (500 points) auto small_dataset_view = raft::make_device_matrix_view( dataset.data_handle(), n_graph, n_dim); @@ -74,13 +74,13 @@ class cagra_graph_smaller_than_dataset_test : public ::testing::Test { auto small_index = cagra::build(res, small_index_params, small_dataset_view); raft::resource::sync_stream(res); - // Step 2: Update to FULL dataset (10000 points) but keep small graph (5000 nodes) - // This creates the exact bug scenario: dataset.size=10000, graph.extent(0)=5000 + // Step 2: Update to FULL dataset (1000 points) but keep small graph (500 nodes) + // This creates the exact bug scenario: dataset.size=1000, graph.extent(0)=500 small_index.update_dataset(res, raft::make_const_mdspan(dataset.view())); // Verify the mismatch - THIS IS THE BUG SCENARIO! - ASSERT_EQ(small_index.graph().extent(0), n_graph); // Graph has 5000 nodes - ASSERT_EQ(small_index.size(), n_dataset); // Dataset has 10000 points + ASSERT_EQ(small_index.graph().extent(0), n_graph); // Graph has 500 nodes + ASSERT_EQ(small_index.size(), n_dataset); // Dataset has 1000 points ASSERT_NE(small_index.graph().extent(0), small_index.size()); // Mismatch! // Create queries @@ -100,8 +100,8 @@ class cagra_graph_smaller_than_dataset_test : public ::testing::Test { search_params.algo = cagra::search_algo::SINGLE_CTA; // THIS SHOULD NOT CRASH OR CAUSE OOB ACCESS - // Before fix: random seeds use dataset.size (10000) -> tries to access graph[7000] -> CRASH! - // After fix: random seeds use graph.extent(0) (5000) -> only accesses graph[0-4999] -> SAFE! + // Before fix: random seeds use dataset.size (1000) -> tries to access graph[700] -> CRASH! + // After fix: random seeds use graph.extent(0) (500) -> only accesses graph[0-499] -> SAFE! cagra::search(res, search_params, small_index, From 8a378cb53319ccac0f6b755aa05f2e4cd78a5f63 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Mon, 10 Aug 2026 05:33:22 -0700 Subject: [PATCH 18/24] Removed dataset shuffle --- .../neighbors/detail/cagra/cagra_build.cuh | 146 +----------------- 1 file changed, 7 insertions(+), 139 deletions(-) diff --git a/cpp/src/neighbors/detail/cagra/cagra_build.cuh b/cpp/src/neighbors/detail/cagra/cagra_build.cuh index 820cbef016..1446bb81a5 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_build.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_build.cuh @@ -62,32 +62,6 @@ namespace cuvs::neighbors::cagra::detail { constexpr double to_mib(size_t bytes) { return static_cast(bytes) / (1 << 20); } constexpr double to_gib(size_t bytes) { return static_cast(bytes) / (1 << 30); } -// Functor to remap indices using a permutation lookup table -template -struct remap_indices_op { - const IdxT* perm; - __host__ __device__ IdxT operator()(IdxT idx) const { return perm[idx]; } -}; - -// Functor to compute scattered output index for graph row reordering -template -struct graph_scatter_index_op { - const IdxT* perm; - int64_t degree; - __host__ __device__ int64_t operator()(int64_t idx) const - { - int64_t row = idx / degree; - int64_t col = idx % degree; - return static_cast(perm[row]) * degree + col; - } -}; - -// Functor to convert int64_t to IdxT -template -struct cast_to_idx_op { - __host__ __device__ IdxT operator()(int64_t v) const { return static_cast(v); } -}; - template void check_graph_degree(size_t& intermediate_degree, size_t& graph_degree, size_t dataset_size) { @@ -2157,6 +2131,12 @@ void search_and_optimize(raft::resources const& res, graph::optimize(res, dev_knn_graph.view(), dev_output_graph.view(), false); } +// Builds a CAGRA graph iteratively, growing the graph while repeatedly running CAGRA's search() +// and optimize(). When compression is enabled the compressed dataset is consumed as-is. +// +// NOTE: This function EXPECTS the dataset to already be shuffled by the caller when a randomized +// build order is desired. It no longer performs any dataset shuffling (or graph unshuffling) +// internally, so the returned graph's node ordering matches the order of the input dataset. template > idx_opt; - // Optional shuffle permutation for randomizing dataset order during build. - // inverse_perm[shuffled_idx] = original_idx - // perm[shuffled_idx] = original_idx, used to unshuffle the graph after build - auto dev_perm = raft::make_device_vector(res, 0); - bool dataset_shuffled = false; - - // Warn if shuffle is requested but compression is not enabled - if (iter_params.shuffle_dataset && !build_compression.has_value()) { - RAFT_LOG_WARN("shuffle_dataset is only supported with compression enabled; ignoring"); - } - if (build_compression.has_value()) { auto start = std::chrono::high_resolution_clock::now(); RAFT_EXPECTS(params.metric == cuvs::distance::DistanceType::L2Expanded, @@ -2310,62 +2279,8 @@ auto iterative_build_graph( auto vpq_dset = cuvs::preprocessing::quantize::pq::vpq_build(res, *build_compression, dev_dataset); - // Optionally shuffle the compressed dataset to break spatial locality - if (iter_params.shuffle_dataset) { - auto shuffle_start = std::chrono::high_resolution_clock::now(); - RAFT_LOG_INFO("Shuffling compressed dataset to randomize build order..."); - - auto stream = raft::resource::get_cuda_stream(res); - const auto n_rows = vpq_dset.data.extent(0); - const auto row_len = vpq_dset.data.extent(1); - - // Generate random permutation: perm[i] = source index for output row i - // i.e., shuffled_data[i] = original_data[perm[i]] - // So perm maps: shuffled_idx -> original_idx - // Use int64_t for permutation to match vpq_dataset's index type - auto dev_perm_i64 = raft::make_device_vector(res, n_rows); - - // Use legacy permute API to generate permutation indices only (out=nullptr, in=nullptr) - // This just fills dev_perm_i64 with a random permutation of [0, n_rows) - raft::random::permute(dev_perm_i64.data_handle(), - static_cast(nullptr), - static_cast(nullptr), - static_cast(row_len), - static_cast(n_rows), - true, - stream); - - // Apply permutation to VPQ data in place: data[i] = original_data[perm[i]]. - // Previously this used an out-of-place gather into a temporary buffer to work around - // an illegal memory access in the in-place gather overload when n_rows * row_len - // exceeded 2^31 (32-bit index overflow). That bug is fixed upstream in raft - // (rapidsai/raft#3059, issue #3055), which cuvs now pins, so the in-place gather is - // safe again and avoids the extra full-size temporary allocation and copy. - raft::matrix::gather(res, vpq_dset.data.view(), raft::make_const_mdspan(dev_perm_i64.view())); - - // Store perm as IdxT for graph unshuffling later - // perm[shuffled_idx] = original_idx - // This is used for: - // 1. Remapping neighbor values: neighbor j (shuffled) -> perm[j] (original) - // 2. Reordering rows: row i (for shuffled node i) -> position perm[i] (original node) - dev_perm = raft::make_device_vector(res, n_rows); - cast_to_idx_op cast_op; - thrust::transform(raft::resource::get_thrust_policy(res), - dev_perm_i64.data_handle(), - dev_perm_i64.data_handle() + n_rows, - dev_perm.data_handle(), - cast_op); - - dataset_shuffled = true; - - auto shuffle_end = std::chrono::high_resolution_clock::now(); - auto shuffle_ms = - std::chrono::duration_cast(shuffle_end - shuffle_start).count(); - RAFT_LOG_INFO("# Dataset shuffle time: %.3lf sec", (double)shuffle_ms / 1000); - } - idx_opt.emplace(res, params.metric); - // Use the (optionally shuffled) compressed dataset built above. + // Use the compressed dataset built above (expected to be pre-shuffled by the caller). idx_opt->update_dataset(res, std::move(vpq_dset)); auto end = std::chrono::high_resolution_clock::now(); auto elapsed_ms = std::chrono::duration_cast(end - start).count(); @@ -2488,53 +2403,6 @@ auto iterative_build_graph( // the index directly (with its VPQ dataset and device-side graph) instead of just the host graph. auto stream = raft::resource::get_cuda_stream(res); - // If the dataset was shuffled, we need to unshuffle the graph: - // Recall: perm[shuffled_idx] = original_idx (stored in dev_perm) - // 1. Remap neighbor indices from shuffled space to original space - // 2. Reorder rows from shuffled order to original order - if (dataset_shuffled) { - auto unshuffle_start = std::chrono::high_resolution_clock::now(); - RAFT_LOG_INFO("Unshuffling graph to restore original dataset ordering..."); - - const auto n_rows = dev_graph.extent(0); - const auto degree = dev_graph.extent(1); - - // Step 1: Remap all neighbor indices using perm - // graph[i][j] contains shuffled index j; we need original index = perm[j] - remap_indices_op remap_op{dev_perm.data_handle()}; - thrust::transform(raft::resource::get_thrust_policy(res), - dev_graph.data_handle(), - dev_graph.data_handle() + n_rows * degree, - dev_graph.data_handle(), - remap_op); - - // Step 2: Reorder rows back to original order - // Row i in dev_graph is for shuffled node i, which is original node perm[i]. - // We want this row to be at position perm[i] in the final graph. - // scatter: output[map[i]] = input[i], so map[i] = perm[i] - auto dev_unshuffled_graph = raft::make_device_matrix(res, n_rows, degree); - - // Use thrust::scatter to reorder: for each row i, place it at position perm[i] - // We scatter row-by-row conceptually, but do it element-wise with computed output indices - graph_scatter_index_op scatter_idx_op{dev_perm.data_handle(), degree}; - auto output_indices = - thrust::make_transform_iterator(thrust::make_counting_iterator(0), scatter_idx_op); - - thrust::scatter(raft::resource::get_thrust_policy(res), - dev_graph.data_handle(), - dev_graph.data_handle() + n_rows * degree, - output_indices, - dev_unshuffled_graph.data_handle()); - - dev_graph = std::move(dev_unshuffled_graph); - - auto unshuffle_end = std::chrono::high_resolution_clock::now(); - auto unshuffle_ms = - std::chrono::duration_cast(unshuffle_end - unshuffle_start) - .count(); - RAFT_LOG_INFO("# Graph unshuffle time: %.3lf sec", (double)unshuffle_ms / 1000); - } - cagra_graph = raft::make_host_matrix(dev_graph.extent(0), dev_graph.extent(1)); raft::copy(cagra_graph.data_handle(), dev_graph.data_handle(), From aa2227b8fb08d1c2718d21c8b0ff0af171258ed2 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Mon, 10 Aug 2026 05:52:09 -0700 Subject: [PATCH 19/24] Removed unused pointer residency helper; use memory_type_from_pointer instead --- cpp/src/neighbors/detail/cagra/utils.hpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/cpp/src/neighbors/detail/cagra/utils.hpp b/cpp/src/neighbors/detail/cagra/utils.hpp index 2313631372..7b31fbdee3 100644 --- a/cpp/src/neighbors/detail/cagra/utils.hpp +++ b/cpp/src/neighbors/detail/cagra/utils.hpp @@ -161,22 +161,6 @@ struct gen_index_msb_1_mask { }; } // namespace utils -template -bool is_ptr_device_accessible(T* ptr) -{ - cudaPointerAttributes attr; - RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr, ptr)); - return attr.devicePointer != nullptr; -} - -template -bool is_ptr_host_accessible(T* ptr) -{ - cudaPointerAttributes attr; - RAFT_CUDA_TRY(cudaPointerGetAttributes(&attr, ptr)); - return attr.hostPointer != nullptr; -} - /** * Utility to sync memory from a host_matrix_view to a device_matrix_view * From d9c6bfd10941470b85c22735834749b363cc5ebb Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Mon, 10 Aug 2026 06:01:04 -0700 Subject: [PATCH 20/24] Pre-commit changes --- c/src/neighbors/brute_force.cpp | 2 +- c/src/neighbors/cagra.cpp | 2 +- c/src/neighbors/ivf_flat.cpp | 2 +- c/src/neighbors/mg_cagra.cpp | 2 +- c/src/neighbors/mg_ivf_flat.cpp | 2 +- c/src/neighbors/mg_ivf_pq.cpp | 2 +- ci/build_cpp.sh | 2 +- ci/build_go.sh | 2 +- ci/build_java.sh | 2 +- ci/build_python.sh | 2 +- ci/build_rust.sh | 2 +- ci/build_wheel_cuvs.sh | 2 +- ci/build_wheel_libcuvs.sh | 2 +- ci/test_cpp.sh | 2 +- ci/test_wheel_cuvs.sh | 2 +- cpp/bench/ann/CMakeLists.txt | 2 +- cpp/bench/ann/src/cuvs/cuvs_benchmark.cu | 2 +- cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu | 2 +- cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h | 2 +- cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu | 2 +- cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h | 2 +- cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp | 2 +- cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h | 2 +- cpp/cmake/modules/compute_matrix_product.cmake | 2 +- cpp/include/cuvs/detail/jit_lto/common_fragments.hpp | 2 +- .../cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp | 2 +- cpp/include/cuvs/neighbors/ivf_pq.hpp | 2 +- cpp/include/cuvs/neighbors/ivf_rabitq.hpp | 2 +- cpp/include/cuvs/neighbors/nn_descent.hpp | 2 +- cpp/include/cuvs/util/file_io.hpp | 2 +- cpp/src/neighbors/brute_force_serialize.cu | 2 +- cpp/src/neighbors/cagra.cuh | 2 +- cpp/src/neighbors/detail/cagra/cagra_helpers.hpp | 2 +- cpp/src/neighbors/detail/cagra/cagra_serialize.cuh | 2 +- cpp/src/neighbors/detail/cagra/graph_core.cuh | 2 +- .../detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh | 2 +- .../detail/cagra/jit_lto_kernels/search_multi_jit.cuh | 2 +- cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in | 2 +- .../detail/cagra/search_multi_cta_kernel_launcher_jit.cuh | 2 +- .../detail/cagra/search_multi_kernel_launcher_jit.cuh | 2 +- cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in | 2 +- cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp | 2 +- cpp/src/neighbors/detail/hnsw.hpp | 2 +- cpp/src/neighbors/ivf_pq_index.cu | 2 +- cpp/src/neighbors/ivf_rabitq.cu | 2 +- cpp/src/neighbors/ivf_rabitq/defines.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh | 2 +- .../neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh | 2 +- .../ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu | 2 +- .../ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu | 2 +- .../bitwise_block_sort_emit_topk_kernel.cu.in | 2 +- .../jit_lto_kernels/bitwise_emit_distances_kernel.cu.in | 2 +- cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh | 2 +- .../compute_bitwise_quantized_ip_for_vec_kernel.cu.in | 2 +- .../compute_inner_products_with_bitwise_block_sort_impl.cuh | 2 +- ...pute_inner_products_with_bitwise_block_sort_kernel.cu.in | 2 +- ...mpute_inner_products_with_bitwise_block_sort_planner.hpp | 2 +- .../compute_inner_products_with_bitwise_impl.cuh | 2 +- .../compute_inner_products_with_bitwise_kernel.cu.in | 2 +- .../compute_inner_products_with_bitwise_planner.hpp | 2 +- ...ompute_inner_products_with_lut16_opt_block_sort_impl.cuh | 2 +- ...te_inner_products_with_lut16_opt_block_sort_kernel.cu.in | 2 +- ...ute_inner_products_with_lut16_opt_block_sort_planner.hpp | 2 +- .../compute_inner_products_with_lut16_opt_impl.cuh | 2 +- .../compute_inner_products_with_lut16_opt_kernel.cu.in | 2 +- .../compute_inner_products_with_lut16_opt_planner.hpp | 2 +- .../compute_inner_products_with_lut_block_sort_impl.cuh | 2 +- .../compute_inner_products_with_lut_block_sort_kernel.cu.in | 2 +- .../compute_inner_products_with_lut_block_sort_planner.hpp | 2 +- .../compute_inner_products_with_lut_impl.cuh | 2 +- .../compute_inner_products_with_lut_kernel.cu.in | 2 +- .../compute_inner_products_with_lut_planner.hpp | 2 +- .../jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in | 2 +- .../ivf_rabitq/jit_lto_kernels/device_functions.cuh | 2 +- .../ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in | 2 +- cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp | 2 +- .../ivf_rabitq/jit_lto_kernels/launcher_factory.hpp | 2 +- .../jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in | 2 +- .../jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in | 2 +- .../jit_lto_kernels/lut_emit_distances_kernel.cu.in | 2 +- cpp/src/neighbors/ivf_rabitq/utils/IO.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/utils/memory.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu | 2 +- cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/utils/space.hpp | 2 +- cpp/src/neighbors/mg/snmg.cuh | 2 +- cpp/src/neighbors/nn_descent.cu | 2 +- cpp/tests/neighbors/ann_cagra.cuh | 6 +++--- cpp/tests/neighbors/ann_cagra/test_filter_udf.cu | 2 +- cpp/tests/neighbors/ann_hnsw_ace.cuh | 2 +- cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu | 2 +- cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu | 2 +- cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu | 2 +- cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu | 2 +- cpp/tests/neighbors/ann_ivf_rabitq.cuh | 2 +- cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu | 2 +- examples/build.sh | 2 +- examples/cpp/src/cagra_filter_udf_example.cu | 2 +- examples/cpp/src/cagra_hnsw_ace_build.cu | 2 +- examples/cpp/src/hnsw_openai_example.cu | 2 +- python/cuvs/cuvs/tests/test_cagra_ace.py | 2 +- python/cuvs/cuvs/tests/test_hnsw_ace.py | 2 +- 113 files changed, 115 insertions(+), 115 deletions(-) diff --git a/c/src/neighbors/brute_force.cpp b/c/src/neighbors/brute_force.cpp index 081f433a3e..d926a56cd4 100644 --- a/c/src/neighbors/brute_force.cpp +++ b/c/src/neighbors/brute_force.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/cagra.cpp b/c/src/neighbors/cagra.cpp index 004b810c78..a01f7b051f 100644 --- a/c/src/neighbors/cagra.cpp +++ b/c/src/neighbors/cagra.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/ivf_flat.cpp b/c/src/neighbors/ivf_flat.cpp index 729c810aa0..7ae9073f9e 100644 --- a/c/src/neighbors/ivf_flat.cpp +++ b/c/src/neighbors/ivf_flat.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/mg_cagra.cpp b/c/src/neighbors/mg_cagra.cpp index 495eff8a34..30be2f76a8 100644 --- a/c/src/neighbors/mg_cagra.cpp +++ b/c/src/neighbors/mg_cagra.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/mg_ivf_flat.cpp b/c/src/neighbors/mg_ivf_flat.cpp index 4e1b2883ea..cdf261a049 100644 --- a/c/src/neighbors/mg_ivf_flat.cpp +++ b/c/src/neighbors/mg_ivf_flat.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/mg_ivf_pq.cpp b/c/src/neighbors/mg_ivf_pq.cpp index 41aa323138..8570f07c8b 100644 --- a/c/src/neighbors/mg_ivf_pq.cpp +++ b/c/src/neighbors/mg_ivf_pq.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index bb497692bd..cda3d10bd9 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_go.sh b/ci/build_go.sh index 056f40b345..e04f6f1d8a 100755 --- a/ci/build_go.sh +++ b/ci/build_go.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_java.sh b/ci/build_java.sh index 2e363bb452..167c642dd2 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_python.sh b/ci/build_python.sh index 6823cbbec5..34be953de5 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_rust.sh b/ci/build_rust.sh index e9218a8adc..e84e454e05 100755 --- a/ci/build_rust.sh +++ b/ci/build_rust.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_wheel_cuvs.sh b/ci/build_wheel_cuvs.sh index 2acbab8a2d..2e174bc87e 100755 --- a/ci/build_wheel_cuvs.sh +++ b/ci/build_wheel_cuvs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_wheel_libcuvs.sh b/ci/build_wheel_libcuvs.sh index e012935749..9bc1ac0a37 100755 --- a/ci/build_wheel_libcuvs.sh +++ b/ci/build_wheel_libcuvs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 0905cd64f0..be62cfc754 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/test_wheel_cuvs.sh b/ci/test_wheel_cuvs.sh index 36fefdf852..227857dd8c 100755 --- a/ci/test_wheel_cuvs.sh +++ b/ci/test_wheel_cuvs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt index 90d23d9aef..80f116f586 100644 --- a/cpp/bench/ann/CMakeLists.txt +++ b/cpp/bench/ann/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= diff --git a/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu b/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu index 3056ddc365..1a334924ec 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu +++ b/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu index c903b39fcc..8c5854051d 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h index db618f6559..c7733c293e 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu b/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu index 1c6772d994..e3c38025c7 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu +++ b/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "cuvs_ivf_rabitq_wrapper.h" diff --git a/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h index ca8f77b808..542f0bc6dd 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp index 1a99e56028..f40ad67a63 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp +++ b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h index 282d57dc2e..cf2bb7608e 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h +++ b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/cmake/modules/compute_matrix_product.cmake b/cpp/cmake/modules/compute_matrix_product.cmake index 82a34f9242..6d81821b88 100644 --- a/cpp/cmake/modules/compute_matrix_product.cmake +++ b/cpp/cmake/modules/compute_matrix_product.cmake @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= diff --git a/cpp/include/cuvs/detail/jit_lto/common_fragments.hpp b/cpp/include/cuvs/detail/jit_lto/common_fragments.hpp index ef2a8e6002..a55052c4a6 100644 --- a/cpp/include/cuvs/detail/jit_lto/common_fragments.hpp +++ b/cpp/include/cuvs/detail/jit_lto/common_fragments.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp b/cpp/include/cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp index 30885947b5..2b2f3db5a7 100644 --- a/cpp/include/cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp +++ b/cpp/include/cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/neighbors/ivf_pq.hpp b/cpp/include/cuvs/neighbors/ivf_pq.hpp index 57f8a258fb..686c3ff108 100644 --- a/cpp/include/cuvs/neighbors/ivf_pq.hpp +++ b/cpp/include/cuvs/neighbors/ivf_pq.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/neighbors/ivf_rabitq.hpp b/cpp/include/cuvs/neighbors/ivf_rabitq.hpp index d26bc04022..c33d466613 100644 --- a/cpp/include/cuvs/neighbors/ivf_rabitq.hpp +++ b/cpp/include/cuvs/neighbors/ivf_rabitq.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/neighbors/nn_descent.hpp b/cpp/include/cuvs/neighbors/nn_descent.hpp index 4c031049e2..929a099cf1 100644 --- a/cpp/include/cuvs/neighbors/nn_descent.hpp +++ b/cpp/include/cuvs/neighbors/nn_descent.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/util/file_io.hpp b/cpp/include/cuvs/util/file_io.hpp index a7d67ec2c0..b0afeed732 100644 --- a/cpp/include/cuvs/util/file_io.hpp +++ b/cpp/include/cuvs/util/file_io.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/neighbors/brute_force_serialize.cu b/cpp/src/neighbors/brute_force_serialize.cu index 1b7595ee11..e3a4a2c041 100644 --- a/cpp/src/neighbors/brute_force_serialize.cu +++ b/cpp/src/neighbors/brute_force_serialize.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/cagra.cuh b/cpp/src/neighbors/cagra.cuh index ee87c2c0ab..34b2e72f90 100644 --- a/cpp/src/neighbors/cagra.cuh +++ b/cpp/src/neighbors/cagra.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/cagra_helpers.hpp b/cpp/src/neighbors/detail/cagra/cagra_helpers.hpp index ee78930970..5c6a63f9fc 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_helpers.hpp +++ b/cpp/src/neighbors/detail/cagra/cagra_helpers.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/neighbors/detail/cagra/cagra_serialize.cuh b/cpp/src/neighbors/detail/cagra/cagra_serialize.cuh index f106b82500..e80c6b6932 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_serialize.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_serialize.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/graph_core.cuh b/cpp/src/neighbors/detail/cagra/graph_core.cuh index 52b4542798..1762bdbfb0 100644 --- a/cpp/src/neighbors/detail/cagra/graph_core.cuh +++ b/cpp/src/neighbors/detail/cagra/graph_core.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh index 4c4f2e4f62..0e9e981e69 100644 --- a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh +++ b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_jit.cuh b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_jit.cuh index a714ced5c2..02cec6ee5e 100644 --- a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_jit.cuh +++ b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_jit.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in b/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in index 7c642fe406..4e94922566 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel_launcher_jit.cuh b/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel_launcher_jit.cuh index 8a673405b7..ff1064f24c 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel_launcher_jit.cuh +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel_launcher_jit.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/search_multi_kernel_launcher_jit.cuh b/cpp/src/neighbors/detail/cagra/search_multi_kernel_launcher_jit.cuh index bc341b9082..549474d045 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_kernel_launcher_jit.cuh +++ b/cpp/src/neighbors/detail/cagra/search_multi_kernel_launcher_jit.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in b/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in index 4616a9652b..7869e4f05e 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp b/cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp index e5157ffa6a..e448bebeb4 100644 --- a/cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp +++ b/cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/hnsw.hpp b/cpp/src/neighbors/detail/hnsw.hpp index 88580de929..649886f924 100644 --- a/cpp/src/neighbors/detail/hnsw.hpp +++ b/cpp/src/neighbors/detail/hnsw.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_pq_index.cu b/cpp/src/neighbors/ivf_pq_index.cu index 28b985eec8..114b98bf0c 100644 --- a/cpp/src/neighbors/ivf_pq_index.cu +++ b/cpp/src/neighbors/ivf_pq_index.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq.cu b/cpp/src/neighbors/ivf_rabitq.cu index d5572d4039..14a9678bad 100644 --- a/cpp/src/neighbors/ivf_rabitq.cu +++ b/cpp/src/neighbors/ivf_rabitq.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/defines.hpp b/cpp/src/neighbors/ivf_rabitq/defines.hpp index f91c06a1ff..aac296cb12 100644 --- a/cpp/src/neighbors/ivf_rabitq/defines.hpp +++ b/cpp/src/neighbors/ivf_rabitq/defines.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu index 6755bfc3de..6e1e2c7d18 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh index 0640148814..135117d4a9 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu index 49070acbcf..c071fb9a0a 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh index 75ddaee865..d275c84eb3 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu index af0876acca..ea40f02931 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh index 90ef71ed12..21aaf414ac 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu index 59a3e575d9..c9c0a1d275 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh index 8db4bb464c..b7a485b7be 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu index 22bb74c682..2c81c40e85 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh index 73557f57ea..68ca342370 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh index 1e004d9164..125a759ca0 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu index 92906fb0e6..fde565e0e9 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu index 8357af5fbb..bee53b54a2 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_block_sort_emit_topk_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_block_sort_emit_topk_kernel.cu.in index d8dcff324d..6a9b0d9576 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_block_sort_emit_topk_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_block_sort_emit_topk_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_emit_distances_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_emit_distances_kernel.cu.in index 20c5aec6a1..fb6a130512 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_emit_distances_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_emit_distances_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh index a16058c2af..fb78d0d560 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_bitwise_quantized_ip_for_vec_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_bitwise_quantized_ip_for_vec_kernel.cu.in index e4e2aa6d79..d5a63ee313 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_bitwise_quantized_ip_for_vec_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_bitwise_quantized_ip_for_vec_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_impl.cuh index a577ec44ce..e3b2da9733 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_kernel.cu.in index 7bbdc24139..785aa22b26 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_planner.hpp index dbe6ec5fd4..b126e723cc 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_impl.cuh index 4ff53b067c..cf87f17314 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_kernel.cu.in index 1f7355c7fe..53897c47db 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_planner.hpp index 3ec4809395..ce7197e7e1 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_impl.cuh index 45c8793b59..cc393f82db 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_kernel.cu.in index 8e45f99c0f..6aa138b917 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_planner.hpp index acf4e27e3a..e71903764b 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_impl.cuh index fc88d79fc4..575766d054 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_kernel.cu.in index 4160aa7a33..713a15b249 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_planner.hpp index 95e0694c9c..8bf18dc94a 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_impl.cuh index 615824621f..23cf3fe49e 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_kernel.cu.in index 014cd12ac2..88a35a8b65 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_planner.hpp index 9db3839880..6ea2783d2c 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_impl.cuh index 1a44453887..12c050d91b 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_kernel.cu.in index 7761879ce0..bb06616f9b 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_planner.hpp index 3559a9bee1..a58f4d8835 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in index 0828b0581a..807899b36e 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/device_functions.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/device_functions.cuh index 7939b8ec13..8f338ab6ea 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/device_functions.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/device_functions.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in index c2b7c21726..3faddca95f 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp index 2314d20b62..db49dc7c72 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/launcher_factory.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/launcher_factory.hpp index d9c94821f1..333243bb45 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/launcher_factory.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/launcher_factory.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in index 24aee45e03..fccc80ed86 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in index 5828027b6d..f177e0cc57 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_emit_distances_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_emit_distances_kernel.cu.in index 28d508e977..0d03cbb15b 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_emit_distances_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_emit_distances_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/IO.hpp b/cpp/src/neighbors/ivf_rabitq/utils/IO.hpp index 0a6b00ba14..8ac69845b0 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/IO.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/IO.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp b/cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp index 8b2934b771..07fb7f1285 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/memory.hpp b/cpp/src/neighbors/ivf_rabitq/utils/memory.hpp index 0050ddee72..d012caa84b 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/memory.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/memory.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh b/cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh index 51d3651b45..287aecc656 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh +++ b/cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu b/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu index 2bb9fb2174..6b82473128 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu +++ b/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp b/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp index 5126cfbdff..1e616ab107 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/space.hpp b/cpp/src/neighbors/ivf_rabitq/utils/space.hpp index df756de7e0..bee35cce2e 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/space.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/space.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/mg/snmg.cuh b/cpp/src/neighbors/mg/snmg.cuh index 43e4aa4471..288a03ebcf 100644 --- a/cpp/src/neighbors/mg/snmg.cuh +++ b/cpp/src/neighbors/mg/snmg.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/nn_descent.cu b/cpp/src/neighbors/nn_descent.cu index eb2541b553..9405d4e608 100644 --- a/cpp/src/neighbors/nn_descent.cu +++ b/cpp/src/neighbors/nn_descent.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_cagra.cuh b/cpp/tests/neighbors/ann_cagra.cuh index f85e36a6ef..84c94eaee7 100644 --- a/cpp/tests/neighbors/ann_cagra.cuh +++ b/cpp/tests/neighbors/ann_cagra.cuh @@ -1628,11 +1628,11 @@ inline std::vector generate_inputs() {100}, {1000}, {1, 3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 768, 1024}, // dim - {16}, // k - {32}, // degree + {16}, // k + {32}, // degree {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT, - graph_build_algo::ITERATIVE_CAGRA_SEARCH}, // Iterative cagra q build + graph_build_algo::ITERATIVE_CAGRA_SEARCH}, // Iterative cagra q build {search_algo::AUTO}, {10}, {0}, diff --git a/cpp/tests/neighbors/ann_cagra/test_filter_udf.cu b/cpp/tests/neighbors/ann_cagra/test_filter_udf.cu index 093727d318..e5dd1f77fc 100644 --- a/cpp/tests/neighbors/ann_cagra/test_filter_udf.cu +++ b/cpp/tests/neighbors/ann_cagra/test_filter_udf.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_hnsw_ace.cuh b/cpp/tests/neighbors/ann_hnsw_ace.cuh index c75b3555f6..30ac24c852 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace.cuh +++ b/cpp/tests/neighbors/ann_hnsw_ace.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu b/cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu index 4cde210d62..da6ba5c969 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu +++ b/cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu b/cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu index d8664d4e14..af167fb4e2 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu +++ b/cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu b/cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu index 4c95192d8a..76f5b8cb71 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu +++ b/cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu b/cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu index 3e4b91e759..433366f05b 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu +++ b/cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_ivf_rabitq.cuh b/cpp/tests/neighbors/ann_ivf_rabitq.cuh index 938f41f846..3c825c9333 100644 --- a/cpp/tests/neighbors/ann_ivf_rabitq.cuh +++ b/cpp/tests/neighbors/ann_ivf_rabitq.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu b/cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu index 2b412f3401..5725856d9a 100644 --- a/cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu +++ b/cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/examples/build.sh b/examples/build.sh index 1be41c01e4..0dc7e2760f 100755 --- a/examples/build.sh +++ b/examples/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cuvs empty project template build script diff --git a/examples/cpp/src/cagra_filter_udf_example.cu b/examples/cpp/src/cagra_filter_udf_example.cu index 0ab42dd580..5da0c10b9e 100644 --- a/examples/cpp/src/cagra_filter_udf_example.cu +++ b/examples/cpp/src/cagra_filter_udf_example.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/examples/cpp/src/cagra_hnsw_ace_build.cu b/examples/cpp/src/cagra_hnsw_ace_build.cu index d23c08e22d..1602b98513 100644 --- a/examples/cpp/src/cagra_hnsw_ace_build.cu +++ b/examples/cpp/src/cagra_hnsw_ace_build.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/examples/cpp/src/hnsw_openai_example.cu b/examples/cpp/src/hnsw_openai_example.cu index abb8346218..3e71f9f1e5 100644 --- a/examples/cpp/src/hnsw_openai_example.cu +++ b/examples/cpp/src/hnsw_openai_example.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/python/cuvs/cuvs/tests/test_cagra_ace.py b/python/cuvs/cuvs/tests/test_cagra_ace.py index c1633e3cad..5ea45781ce 100644 --- a/python/cuvs/cuvs/tests/test_cagra_ace.py +++ b/python/cuvs/cuvs/tests/test_cagra_ace.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # diff --git a/python/cuvs/cuvs/tests/test_hnsw_ace.py b/python/cuvs/cuvs/tests/test_hnsw_ace.py index 183d530e7c..663640e50d 100644 --- a/python/cuvs/cuvs/tests/test_hnsw_ace.py +++ b/python/cuvs/cuvs/tests/test_hnsw_ace.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # From ae89f9f69cc2eb4e9053d89a6b4264dd49be0224 Mon Sep 17 00:00:00 2001 From: Irina Reshodko Date: Mon, 10 Aug 2026 08:22:04 -0700 Subject: [PATCH 21/24] perf(cagra): per-batch VPQ query reconstruction + device-pool graph temporaries in iterative build (~2.7x faster build) --- .../neighbors/detail/cagra/cagra_build.cuh | 168 ++++++++++++------ 1 file changed, 111 insertions(+), 57 deletions(-) diff --git a/cpp/src/neighbors/detail/cagra/cagra_build.cuh b/cpp/src/neighbors/detail/cagra/cagra_build.cuh index 1446bb81a5..df4c8ecb87 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_build.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_build.cuh @@ -2079,56 +2079,115 @@ void reconstruct_vpq_queries(raft::resources const& res, output.data_handle()); } +// Runs CAGRA search for `curr_query_size` queries against `idx` in chunks of `max_chunk_size`, +// stacks the results into a kNN graph, and optimizes it into the next graph (returned). +// +// Query source: +// - `vpq_queries == nullptr`: queries are read directly from `dev_query_view` (uncompressed +// build; the view is a slice of the resident device dataset). +// - `vpq_queries != nullptr`: `dev_query_view` is ignored and each chunk of queries is +// reconstructed on the fly from the VPQ codes into a small reusable scratch buffer, so we +// never materialize the whole (up to N x dim) reconstructed dataset. template -void search_and_optimize(raft::resources const& res, - const cuvs::neighbors::cagra::search_params& search_params, - const index& idx, - raft::device_matrix_view dev_query_view, - raft::device_matrix_view dev_neighbors, - raft::device_matrix_view dev_distances, - raft::device_matrix& dev_output_graph, - size_t curr_query_size, - size_t next_graph_degree, - size_t curr_topk, - uint64_t max_chunk_size) +raft::device_matrix search_and_optimize( + raft::resources const& res, + const cuvs::neighbors::cagra::search_params& search_params, + const index& idx, + raft::device_matrix_view dev_query_view, + raft::device_matrix_view dev_neighbors, + raft::device_matrix_view dev_distances, + raft::device_matrix prev_graph, + const vpq_dataset* vpq_queries, + size_t curr_query_size, + size_t next_graph_degree, + size_t curr_topk, + uint64_t max_chunk_size) { auto stream = raft::resource::get_cuda_stream(res); - auto dev_knn_graph = raft::make_device_matrix(res, curr_query_size, curr_topk); - - auto query_batch = cuvs::spatial::knn::detail::utils::make_batch_load_iterator( - res, - dev_query_view.data_handle(), - static_cast(curr_query_size), - static_cast(dev_query_view.extent(1)), - max_chunk_size, - stream, - raft::resource::get_workspace_resource_ref(res)); - for (const auto& batch : query_batch) { - auto batch_dev_query_view = raft::make_device_matrix_view( - batch.data(), batch.size(), dev_query_view.extent(1)); + // These buffers scale with N (e.g. N * (intermediate_degree+1) for the kNN graph). Allocate them + // from the default device resource (a pool over device memory): allocating from the large + // workspace resource here would use an unpooled managed_memory_resource, paying a synchronous + // cudaMallocManaged/cudaFree every iteration for multi-GB buffers. + auto dev_knn_graph = + raft::make_device_matrix(res, curr_query_size, curr_topk); + + // Query row length: the VPQ dim when reconstructing, otherwise the dataset view's stride. + const int64_t query_dim = + vpq_queries != nullptr ? static_cast(vpq_queries->dim()) : dev_query_view.extent(1); + + // Scratch for one reconstructed chunk (only when compressing). Reused across chunks; safe because + // all reconstruct/search/copy work is serialized on `stream`. + auto batch_queries = + vpq_queries != nullptr + ? raft::make_device_matrix(res, static_cast(max_chunk_size), query_dim) + : raft::make_device_matrix(res, 0, 0); + + auto run_batch = [&](int64_t offset, + int64_t batch_size, + raft::device_matrix_view batch_query_view) { auto batch_dev_neighbors_view = raft::make_device_matrix_view( - dev_neighbors.data_handle(), batch.size(), curr_topk); + dev_neighbors.data_handle(), batch_size, curr_topk); auto batch_dev_distances_view = raft::make_device_matrix_view( - dev_distances.data_handle(), batch.size(), curr_topk); + dev_distances.data_handle(), batch_size, curr_topk); cuvs::neighbors::cagra::search(res, search_params, idx, - batch_dev_query_view, + batch_query_view, batch_dev_neighbors_view, batch_dev_distances_view); - raft::copy(dev_knn_graph.data_handle() + batch.offset() * curr_topk, + raft::copy(dev_knn_graph.data_handle() + offset * curr_topk, batch_dev_neighbors_view.data_handle(), - batch.size() * curr_topk, + batch_size * curr_topk, stream); + }; + + if (vpq_queries != nullptr) { + // Reconstruct-and-search one chunk at a time: reconstruct source rows [offset, offset+bs) into + // the scratch, then search that chunk. + for (int64_t offset = 0; offset < static_cast(curr_query_size); + offset += static_cast(max_chunk_size)) { + const int64_t batch_size = + std::min(static_cast(max_chunk_size), + static_cast(curr_query_size) - offset); + reconstruct_vpq_queries(res, + *vpq_queries, + static_cast(offset), + static_cast(batch_size), + batch_queries.view()); + auto batch_query_view = raft::make_device_matrix_view( + batch_queries.data_handle(), batch_size, query_dim); + run_batch(offset, batch_size, batch_query_view); + } + } else { + auto query_batch = cuvs::spatial::knn::detail::utils::make_batch_load_iterator( + res, + dev_query_view.data_handle(), + static_cast(curr_query_size), + query_dim, + max_chunk_size, + stream, + raft::resource::get_workspace_resource_ref(res)); + for (const auto& batch : query_batch) { + auto batch_query_view = raft::make_device_matrix_view( + batch.data(), static_cast(batch.size()), query_dim); + run_batch( + static_cast(batch.offset()), static_cast(batch.size()), batch_query_view); + } } - dev_output_graph = + // The previous-iteration graph (which `idx` was built on) is no longer needed now that the + // search has produced `dev_knn_graph`. Release it before allocating the full-size output graph + // so we never hold two large graph buffers at once. + prev_graph = raft::make_device_matrix(res, 0, 0); + + auto dev_output_graph = raft::make_device_matrix(res, curr_query_size, next_graph_degree); graph::optimize(res, dev_knn_graph.view(), dev_output_graph.view(), false); + return dev_output_graph; } // Builds a CAGRA graph iteratively, growing the graph while repeatedly running CAGRA's search() @@ -2292,6 +2351,7 @@ auto iterative_build_graph( "# Freed original dataset from device (%.1f MiB); queries will use VPQ reconstruction", to_mib(final_graph_size * dataset_dim * sizeof(T))); } + while (true) { auto start = std::chrono::high_resolution_clock::now(); auto curr_query_size = std::min(2 * curr_graph_size, final_graph_size); @@ -2352,40 +2412,34 @@ auto iterative_build_graph( } const auto& idx = *idx_opt; - // When compression is enabled, reconstruct queries from VPQ codes instead of - // reading from the (freed) original dataset. - auto dev_reconstructed_queries = - build_compression.has_value() - ? raft::make_device_matrix(res, curr_query_size, dataset_dim) - : raft::make_device_matrix(res, 0, 0); + // With compression, search_and_optimize reconstructs queries from the VPQ codes per batch, so + // pass the VPQ dataset and leave the query view empty. Without compression, queries are slices + // of the resident device dataset. + const vpq_dataset* vpq_queries = nullptr; if (build_compression.has_value()) { - auto* vpq_dset = dynamic_cast*>(&idx.data()); - RAFT_EXPECTS(vpq_dset != nullptr, "Expected VPQ dataset in compressed index"); - reconstruct_vpq_queries( - res, *vpq_dset, 0, curr_query_size, dev_reconstructed_queries.view()); + vpq_queries = dynamic_cast*>(&idx.data()); + RAFT_EXPECTS(vpq_queries != nullptr, "Expected VPQ dataset in compressed index"); } auto dev_query_view = build_compression.has_value() - ? raft::make_device_matrix_view( - dev_reconstructed_queries.data_handle(), (int64_t)curr_query_size, dataset_dim) + ? raft::make_device_matrix_view(static_cast(nullptr), 0, 0) : raft::make_device_matrix_view( dev_dataset.data_handle(), (int64_t)curr_query_size, dev_dataset.extent(1)); - auto dev_optimized_graph = raft::make_device_matrix(res, 0, 0); - - search_and_optimize(res, - search_params, - idx, - dev_query_view, - dev_neighbors.view(), - dev_distances.view(), - dev_optimized_graph, - curr_query_size, - next_graph_degree, - curr_topk, - max_chunk_size); - - dev_graph = std::move(dev_optimized_graph); + // Hand the current graph to search_and_optimize so it can release it as soon as the search + // consumes it (before the new output graph is allocated), then take back the new graph. + dev_graph = search_and_optimize(res, + search_params, + idx, + dev_query_view, + dev_neighbors.view(), + dev_distances.view(), + std::move(dev_graph), + vpq_queries, + curr_query_size, + next_graph_degree, + curr_topk, + max_chunk_size); use_device_graph = true; auto end = std::chrono::high_resolution_clock::now(); From 355d240731b803cf3056d0abbaf9876279281452 Mon Sep 17 00:00:00 2001 From: aamijar Date: Mon, 10 Aug 2026 21:59:18 +0000 Subject: [PATCH 22/24] Revert "Pre-commit changes" This reverts commit d9c6bfd10941470b85c22735834749b363cc5ebb. --- c/src/neighbors/brute_force.cpp | 2 +- c/src/neighbors/cagra.cpp | 2 +- c/src/neighbors/ivf_flat.cpp | 2 +- c/src/neighbors/mg_cagra.cpp | 2 +- c/src/neighbors/mg_ivf_flat.cpp | 2 +- c/src/neighbors/mg_ivf_pq.cpp | 2 +- ci/build_cpp.sh | 2 +- ci/build_go.sh | 2 +- ci/build_java.sh | 2 +- ci/build_python.sh | 2 +- ci/build_rust.sh | 2 +- ci/build_wheel_cuvs.sh | 2 +- ci/build_wheel_libcuvs.sh | 2 +- ci/test_cpp.sh | 2 +- ci/test_wheel_cuvs.sh | 2 +- cpp/bench/ann/CMakeLists.txt | 2 +- cpp/bench/ann/src/cuvs/cuvs_benchmark.cu | 2 +- cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu | 2 +- cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h | 2 +- cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu | 2 +- cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h | 2 +- cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp | 2 +- cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h | 2 +- cpp/cmake/modules/compute_matrix_product.cmake | 2 +- cpp/include/cuvs/detail/jit_lto/common_fragments.hpp | 2 +- .../cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp | 2 +- cpp/include/cuvs/neighbors/ivf_pq.hpp | 2 +- cpp/include/cuvs/neighbors/ivf_rabitq.hpp | 2 +- cpp/include/cuvs/neighbors/nn_descent.hpp | 2 +- cpp/include/cuvs/util/file_io.hpp | 2 +- cpp/src/neighbors/brute_force_serialize.cu | 2 +- cpp/src/neighbors/cagra.cuh | 2 +- cpp/src/neighbors/detail/cagra/cagra_helpers.hpp | 2 +- cpp/src/neighbors/detail/cagra/cagra_serialize.cuh | 2 +- cpp/src/neighbors/detail/cagra/graph_core.cuh | 2 +- .../detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh | 2 +- .../detail/cagra/jit_lto_kernels/search_multi_jit.cuh | 2 +- cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in | 2 +- .../detail/cagra/search_multi_cta_kernel_launcher_jit.cuh | 2 +- .../detail/cagra/search_multi_kernel_launcher_jit.cuh | 2 +- cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in | 2 +- cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp | 2 +- cpp/src/neighbors/detail/hnsw.hpp | 2 +- cpp/src/neighbors/ivf_pq_index.cu | 2 +- cpp/src/neighbors/ivf_rabitq.cu | 2 +- cpp/src/neighbors/ivf_rabitq/defines.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu | 2 +- cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh | 2 +- .../neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh | 2 +- .../ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu | 2 +- .../ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu | 2 +- .../bitwise_block_sort_emit_topk_kernel.cu.in | 2 +- .../jit_lto_kernels/bitwise_emit_distances_kernel.cu.in | 2 +- cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh | 2 +- .../compute_bitwise_quantized_ip_for_vec_kernel.cu.in | 2 +- .../compute_inner_products_with_bitwise_block_sort_impl.cuh | 2 +- ...pute_inner_products_with_bitwise_block_sort_kernel.cu.in | 2 +- ...mpute_inner_products_with_bitwise_block_sort_planner.hpp | 2 +- .../compute_inner_products_with_bitwise_impl.cuh | 2 +- .../compute_inner_products_with_bitwise_kernel.cu.in | 2 +- .../compute_inner_products_with_bitwise_planner.hpp | 2 +- ...ompute_inner_products_with_lut16_opt_block_sort_impl.cuh | 2 +- ...te_inner_products_with_lut16_opt_block_sort_kernel.cu.in | 2 +- ...ute_inner_products_with_lut16_opt_block_sort_planner.hpp | 2 +- .../compute_inner_products_with_lut16_opt_impl.cuh | 2 +- .../compute_inner_products_with_lut16_opt_kernel.cu.in | 2 +- .../compute_inner_products_with_lut16_opt_planner.hpp | 2 +- .../compute_inner_products_with_lut_block_sort_impl.cuh | 2 +- .../compute_inner_products_with_lut_block_sort_kernel.cu.in | 2 +- .../compute_inner_products_with_lut_block_sort_planner.hpp | 2 +- .../compute_inner_products_with_lut_impl.cuh | 2 +- .../compute_inner_products_with_lut_kernel.cu.in | 2 +- .../compute_inner_products_with_lut_planner.hpp | 2 +- .../jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in | 2 +- .../ivf_rabitq/jit_lto_kernels/device_functions.cuh | 2 +- .../ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in | 2 +- cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp | 2 +- .../ivf_rabitq/jit_lto_kernels/launcher_factory.hpp | 2 +- .../jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in | 2 +- .../jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in | 2 +- .../jit_lto_kernels/lut_emit_distances_kernel.cu.in | 2 +- cpp/src/neighbors/ivf_rabitq/utils/IO.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/utils/memory.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh | 2 +- cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu | 2 +- cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp | 2 +- cpp/src/neighbors/ivf_rabitq/utils/space.hpp | 2 +- cpp/src/neighbors/mg/snmg.cuh | 2 +- cpp/src/neighbors/nn_descent.cu | 2 +- cpp/tests/neighbors/ann_cagra.cuh | 6 +++--- cpp/tests/neighbors/ann_cagra/test_filter_udf.cu | 2 +- cpp/tests/neighbors/ann_hnsw_ace.cuh | 2 +- cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu | 2 +- cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu | 2 +- cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu | 2 +- cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu | 2 +- cpp/tests/neighbors/ann_ivf_rabitq.cuh | 2 +- cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu | 2 +- examples/build.sh | 2 +- examples/cpp/src/cagra_filter_udf_example.cu | 2 +- examples/cpp/src/cagra_hnsw_ace_build.cu | 2 +- examples/cpp/src/hnsw_openai_example.cu | 2 +- python/cuvs/cuvs/tests/test_cagra_ace.py | 2 +- python/cuvs/cuvs/tests/test_hnsw_ace.py | 2 +- 113 files changed, 115 insertions(+), 115 deletions(-) diff --git a/c/src/neighbors/brute_force.cpp b/c/src/neighbors/brute_force.cpp index d926a56cd4..081f433a3e 100644 --- a/c/src/neighbors/brute_force.cpp +++ b/c/src/neighbors/brute_force.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/cagra.cpp b/c/src/neighbors/cagra.cpp index a01f7b051f..004b810c78 100644 --- a/c/src/neighbors/cagra.cpp +++ b/c/src/neighbors/cagra.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/ivf_flat.cpp b/c/src/neighbors/ivf_flat.cpp index 7ae9073f9e..729c810aa0 100644 --- a/c/src/neighbors/ivf_flat.cpp +++ b/c/src/neighbors/ivf_flat.cpp @@ -1,6 +1,6 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/mg_cagra.cpp b/c/src/neighbors/mg_cagra.cpp index 30be2f76a8..495eff8a34 100644 --- a/c/src/neighbors/mg_cagra.cpp +++ b/c/src/neighbors/mg_cagra.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/mg_ivf_flat.cpp b/c/src/neighbors/mg_ivf_flat.cpp index cdf261a049..4e1b2883ea 100644 --- a/c/src/neighbors/mg_ivf_flat.cpp +++ b/c/src/neighbors/mg_ivf_flat.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/c/src/neighbors/mg_ivf_pq.cpp b/c/src/neighbors/mg_ivf_pq.cpp index 8570f07c8b..41aa323138 100644 --- a/c/src/neighbors/mg_ivf_pq.cpp +++ b/c/src/neighbors/mg_ivf_pq.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index cda3d10bd9..bb497692bd 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_go.sh b/ci/build_go.sh index e04f6f1d8a..056f40b345 100755 --- a/ci/build_go.sh +++ b/ci/build_go.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_java.sh b/ci/build_java.sh index 167c642dd2..2e363bb452 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_python.sh b/ci/build_python.sh index 34be953de5..6823cbbec5 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_rust.sh b/ci/build_rust.sh index e84e454e05..e9218a8adc 100755 --- a/ci/build_rust.sh +++ b/ci/build_rust.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_wheel_cuvs.sh b/ci/build_wheel_cuvs.sh index 2e174bc87e..2acbab8a2d 100755 --- a/ci/build_wheel_cuvs.sh +++ b/ci/build_wheel_cuvs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/build_wheel_libcuvs.sh b/ci/build_wheel_libcuvs.sh index 9bc1ac0a37..e012935749 100755 --- a/ci/build_wheel_libcuvs.sh +++ b/ci/build_wheel_libcuvs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index be62cfc754..0905cd64f0 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/ci/test_wheel_cuvs.sh b/ci/test_wheel_cuvs.sh index 227857dd8c..36fefdf852 100755 --- a/ci/test_wheel_cuvs.sh +++ b/ci/test_wheel_cuvs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt index 80f116f586..90d23d9aef 100644 --- a/cpp/bench/ann/CMakeLists.txt +++ b/cpp/bench/ann/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= diff --git a/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu b/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu index 1a334924ec..3056ddc365 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu +++ b/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu index 8c5854051d..c903b39fcc 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h index c7733c293e..db618f6559 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu b/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu index e3c38025c7..1c6772d994 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu +++ b/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #include "cuvs_ivf_rabitq_wrapper.h" diff --git a/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h index 542f0bc6dd..ca8f77b808 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_ivf_rabitq_wrapper.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp index f40ad67a63..1a99e56028 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp +++ b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h index cf2bb7608e..282d57dc2e 100644 --- a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h +++ b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/cmake/modules/compute_matrix_product.cmake b/cpp/cmake/modules/compute_matrix_product.cmake index 6d81821b88..82a34f9242 100644 --- a/cpp/cmake/modules/compute_matrix_product.cmake +++ b/cpp/cmake/modules/compute_matrix_product.cmake @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= diff --git a/cpp/include/cuvs/detail/jit_lto/common_fragments.hpp b/cpp/include/cuvs/detail/jit_lto/common_fragments.hpp index a55052c4a6..ef2a8e6002 100644 --- a/cpp/include/cuvs/detail/jit_lto/common_fragments.hpp +++ b/cpp/include/cuvs/detail/jit_lto/common_fragments.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp b/cpp/include/cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp index 2b2f3db5a7..30885947b5 100644 --- a/cpp/include/cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp +++ b/cpp/include/cuvs/detail/jit_lto/ivf_rabitq/ivf_rabitq_fragments.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/neighbors/ivf_pq.hpp b/cpp/include/cuvs/neighbors/ivf_pq.hpp index 686c3ff108..57f8a258fb 100644 --- a/cpp/include/cuvs/neighbors/ivf_pq.hpp +++ b/cpp/include/cuvs/neighbors/ivf_pq.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/neighbors/ivf_rabitq.hpp b/cpp/include/cuvs/neighbors/ivf_rabitq.hpp index c33d466613..d26bc04022 100644 --- a/cpp/include/cuvs/neighbors/ivf_rabitq.hpp +++ b/cpp/include/cuvs/neighbors/ivf_rabitq.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/neighbors/nn_descent.hpp b/cpp/include/cuvs/neighbors/nn_descent.hpp index 929a099cf1..4c031049e2 100644 --- a/cpp/include/cuvs/neighbors/nn_descent.hpp +++ b/cpp/include/cuvs/neighbors/nn_descent.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cuvs/util/file_io.hpp b/cpp/include/cuvs/util/file_io.hpp index b0afeed732..a7d67ec2c0 100644 --- a/cpp/include/cuvs/util/file_io.hpp +++ b/cpp/include/cuvs/util/file_io.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/neighbors/brute_force_serialize.cu b/cpp/src/neighbors/brute_force_serialize.cu index e3a4a2c041..1b7595ee11 100644 --- a/cpp/src/neighbors/brute_force_serialize.cu +++ b/cpp/src/neighbors/brute_force_serialize.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/cagra.cuh b/cpp/src/neighbors/cagra.cuh index 34b2e72f90..ee87c2c0ab 100644 --- a/cpp/src/neighbors/cagra.cuh +++ b/cpp/src/neighbors/cagra.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/cagra_helpers.hpp b/cpp/src/neighbors/detail/cagra/cagra_helpers.hpp index 5c6a63f9fc..ee78930970 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_helpers.hpp +++ b/cpp/src/neighbors/detail/cagra/cagra_helpers.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/neighbors/detail/cagra/cagra_serialize.cuh b/cpp/src/neighbors/detail/cagra/cagra_serialize.cuh index e80c6b6932..f106b82500 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_serialize.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_serialize.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/graph_core.cuh b/cpp/src/neighbors/detail/cagra/graph_core.cuh index 1762bdbfb0..52b4542798 100644 --- a/cpp/src/neighbors/detail/cagra/graph_core.cuh +++ b/cpp/src/neighbors/detail/cagra/graph_core.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh index 0e9e981e69..4c4f2e4f62 100644 --- a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh +++ b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_cta_jit.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_jit.cuh b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_jit.cuh index 02cec6ee5e..a714ced5c2 100644 --- a/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_jit.cuh +++ b/cpp/src/neighbors/detail/cagra/jit_lto_kernels/search_multi_jit.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in b/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in index 4e94922566..7c642fe406 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_inst.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel_launcher_jit.cuh b/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel_launcher_jit.cuh index ff1064f24c..8a673405b7 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel_launcher_jit.cuh +++ b/cpp/src/neighbors/detail/cagra/search_multi_cta_kernel_launcher_jit.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/search_multi_kernel_launcher_jit.cuh b/cpp/src/neighbors/detail/cagra/search_multi_kernel_launcher_jit.cuh index 549474d045..bc341b9082 100644 --- a/cpp/src/neighbors/detail/cagra/search_multi_kernel_launcher_jit.cuh +++ b/cpp/src/neighbors/detail/cagra/search_multi_kernel_launcher_jit.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in b/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in index 7869e4f05e..4616a9652b 100644 --- a/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in +++ b/cpp/src/neighbors/detail/cagra/search_single_cta_inst.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp b/cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp index e448bebeb4..e5157ffa6a 100644 --- a/cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp +++ b/cpp/src/neighbors/detail/cagra/shared_launcher_jit.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/detail/hnsw.hpp b/cpp/src/neighbors/detail/hnsw.hpp index 649886f924..88580de929 100644 --- a/cpp/src/neighbors/detail/hnsw.hpp +++ b/cpp/src/neighbors/detail/hnsw.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_pq_index.cu b/cpp/src/neighbors/ivf_pq_index.cu index 114b98bf0c..28b985eec8 100644 --- a/cpp/src/neighbors/ivf_pq_index.cu +++ b/cpp/src/neighbors/ivf_pq_index.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq.cu b/cpp/src/neighbors/ivf_rabitq.cu index 14a9678bad..d5572d4039 100644 --- a/cpp/src/neighbors/ivf_rabitq.cu +++ b/cpp/src/neighbors/ivf_rabitq.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/defines.hpp b/cpp/src/neighbors/ivf_rabitq/defines.hpp index aac296cb12..f91c06a1ff 100644 --- a/cpp/src/neighbors/ivf_rabitq/defines.hpp +++ b/cpp/src/neighbors/ivf_rabitq/defines.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu index 6e1e2c7d18..6755bfc3de 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh index 135117d4a9..0640148814 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/initializer_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu index c071fb9a0a..49070acbcf 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh index d275c84eb3..75ddaee865 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/ivf_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu index ea40f02931..af0876acca 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh index 21aaf414ac..90ef71ed12 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/quantizer_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu index c9c0a1d275..59a3e575d9 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh index b7a485b7be..8db4bb464c 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/rotator_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu index 2c81c40e85..22bb74c682 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh index 68ca342370..73557f57ea 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh index 125a759ca0..1e004d9164 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_common.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu index fde565e0e9..92906fb0e6 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_quantize_query.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu index bee53b54a2..8357af5fbb 100644 --- a/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu +++ b/cpp/src/neighbors/ivf_rabitq/gpu_index/searcher_gpu_shared_mem_opt.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_block_sort_emit_topk_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_block_sort_emit_topk_kernel.cu.in index 6a9b0d9576..d8dcff324d 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_block_sort_emit_topk_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_block_sort_emit_topk_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_emit_distances_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_emit_distances_kernel.cu.in index fb6a130512..20c5aec6a1 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_emit_distances_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/bitwise_emit_distances_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh index fb78d0d560..a16058c2af 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/block_sort.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_bitwise_quantized_ip_for_vec_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_bitwise_quantized_ip_for_vec_kernel.cu.in index d5a63ee313..e4e2aa6d79 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_bitwise_quantized_ip_for_vec_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_bitwise_quantized_ip_for_vec_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_impl.cuh index e3b2da9733..a577ec44ce 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_kernel.cu.in index 785aa22b26..7bbdc24139 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_planner.hpp index b126e723cc..dbe6ec5fd4 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_block_sort_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_impl.cuh index cf87f17314..4ff53b067c 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_kernel.cu.in index 53897c47db..1f7355c7fe 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_planner.hpp index ce7197e7e1..3ec4809395 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_bitwise_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_impl.cuh index cc393f82db..45c8793b59 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_kernel.cu.in index 6aa138b917..8e45f99c0f 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_planner.hpp index e71903764b..acf4e27e3a 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_block_sort_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_impl.cuh index 575766d054..fc88d79fc4 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_kernel.cu.in index 713a15b249..4160aa7a33 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_planner.hpp index 8bf18dc94a..95e0694c9c 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut16_opt_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_impl.cuh index 23cf3fe49e..615824621f 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_kernel.cu.in index 88a35a8b65..014cd12ac2 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_planner.hpp index 6ea2783d2c..9db3839880 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_block_sort_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_impl.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_impl.cuh index 12c050d91b..1a44453887 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_impl.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_kernel.cu.in index bb06616f9b..7761879ce0 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_planner.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_planner.hpp index a58f4d8835..3559a9bee1 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_planner.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_inner_products_with_lut_planner.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in index 807899b36e..0828b0581a 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/compute_lut_ip_for_vec_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/device_functions.cuh b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/device_functions.cuh index 8f338ab6ea..7939b8ec13 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/device_functions.cuh +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/device_functions.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in index 3faddca95f..c2b7c21726 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/extract_code_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp index db49dc7c72..2314d20b62 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/kernel_def.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/launcher_factory.hpp b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/launcher_factory.hpp index 333243bb45..d9c94821f1 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/launcher_factory.hpp +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/launcher_factory.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in index fccc80ed86..24aee45e03 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut16_opt_emit_distances_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in index f177e0cc57..5828027b6d 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_block_sort_emit_topk_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_emit_distances_kernel.cu.in b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_emit_distances_kernel.cu.in index 0d03cbb15b..28d508e977 100644 --- a/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_emit_distances_kernel.cu.in +++ b/cpp/src/neighbors/ivf_rabitq/jit_lto_kernels/lut_emit_distances_kernel.cu.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/IO.hpp b/cpp/src/neighbors/ivf_rabitq/utils/IO.hpp index 8ac69845b0..0a6b00ba14 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/IO.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/IO.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp b/cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp index 07fb7f1285..8b2934b771 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/StopW.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/memory.hpp b/cpp/src/neighbors/ivf_rabitq/utils/memory.hpp index d012caa84b..0050ddee72 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/memory.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/memory.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh b/cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh index 287aecc656..51d3651b45 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh +++ b/cpp/src/neighbors/ivf_rabitq/utils/reductions.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu b/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu index 6b82473128..2bb9fb2174 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu +++ b/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp b/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp index 1e616ab107..5126cfbdff 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/searcher_gpu_utils.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/ivf_rabitq/utils/space.hpp b/cpp/src/neighbors/ivf_rabitq/utils/space.hpp index bee35cce2e..df756de7e0 100644 --- a/cpp/src/neighbors/ivf_rabitq/utils/space.hpp +++ b/cpp/src/neighbors/ivf_rabitq/utils/space.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/mg/snmg.cuh b/cpp/src/neighbors/mg/snmg.cuh index 288a03ebcf..43e4aa4471 100644 --- a/cpp/src/neighbors/mg/snmg.cuh +++ b/cpp/src/neighbors/mg/snmg.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/neighbors/nn_descent.cu b/cpp/src/neighbors/nn_descent.cu index 9405d4e608..eb2541b553 100644 --- a/cpp/src/neighbors/nn_descent.cu +++ b/cpp/src/neighbors/nn_descent.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_cagra.cuh b/cpp/tests/neighbors/ann_cagra.cuh index 84c94eaee7..f85e36a6ef 100644 --- a/cpp/tests/neighbors/ann_cagra.cuh +++ b/cpp/tests/neighbors/ann_cagra.cuh @@ -1628,11 +1628,11 @@ inline std::vector generate_inputs() {100}, {1000}, {1, 3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 768, 1024}, // dim - {16}, // k - {32}, // degree + {16}, // k + {32}, // degree {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT, - graph_build_algo::ITERATIVE_CAGRA_SEARCH}, // Iterative cagra q build + graph_build_algo::ITERATIVE_CAGRA_SEARCH}, // Iterative cagra q build {search_algo::AUTO}, {10}, {0}, diff --git a/cpp/tests/neighbors/ann_cagra/test_filter_udf.cu b/cpp/tests/neighbors/ann_cagra/test_filter_udf.cu index e5dd1f77fc..093727d318 100644 --- a/cpp/tests/neighbors/ann_cagra/test_filter_udf.cu +++ b/cpp/tests/neighbors/ann_cagra/test_filter_udf.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_hnsw_ace.cuh b/cpp/tests/neighbors/ann_hnsw_ace.cuh index 30ac24c852..c75b3555f6 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace.cuh +++ b/cpp/tests/neighbors/ann_hnsw_ace.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu b/cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu index da6ba5c969..4cde210d62 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu +++ b/cpp/tests/neighbors/ann_hnsw_ace/test_float_uint32_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu b/cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu index af167fb4e2..d8664d4e14 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu +++ b/cpp/tests/neighbors/ann_hnsw_ace/test_half_uint32_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu b/cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu index 76f5b8cb71..4c95192d8a 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu +++ b/cpp/tests/neighbors/ann_hnsw_ace/test_int8_t_uint32_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu b/cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu index 433366f05b..3e4b91e759 100644 --- a/cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu +++ b/cpp/tests/neighbors/ann_hnsw_ace/test_uint8_t_uint32_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/neighbors/ann_ivf_rabitq.cuh b/cpp/tests/neighbors/ann_ivf_rabitq.cuh index 3c825c9333..938f41f846 100644 --- a/cpp/tests/neighbors/ann_ivf_rabitq.cuh +++ b/cpp/tests/neighbors/ann_ivf_rabitq.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu b/cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu index 5725856d9a..2b412f3401 100644 --- a/cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu +++ b/cpp/tests/neighbors/ann_ivf_rabitq/test_float_int64_t.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/examples/build.sh b/examples/build.sh index 0dc7e2760f..1be41c01e4 100755 --- a/examples/build.sh +++ b/examples/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # cuvs empty project template build script diff --git a/examples/cpp/src/cagra_filter_udf_example.cu b/examples/cpp/src/cagra_filter_udf_example.cu index 5da0c10b9e..0ab42dd580 100644 --- a/examples/cpp/src/cagra_filter_udf_example.cu +++ b/examples/cpp/src/cagra_filter_udf_example.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/examples/cpp/src/cagra_hnsw_ace_build.cu b/examples/cpp/src/cagra_hnsw_ace_build.cu index 1602b98513..d23c08e22d 100644 --- a/examples/cpp/src/cagra_hnsw_ace_build.cu +++ b/examples/cpp/src/cagra_hnsw_ace_build.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/examples/cpp/src/hnsw_openai_example.cu b/examples/cpp/src/hnsw_openai_example.cu index 3e71f9f1e5..abb8346218 100644 --- a/examples/cpp/src/hnsw_openai_example.cu +++ b/examples/cpp/src/hnsw_openai_example.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/python/cuvs/cuvs/tests/test_cagra_ace.py b/python/cuvs/cuvs/tests/test_cagra_ace.py index 5ea45781ce..c1633e3cad 100644 --- a/python/cuvs/cuvs/tests/test_cagra_ace.py +++ b/python/cuvs/cuvs/tests/test_cagra_ace.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # diff --git a/python/cuvs/cuvs/tests/test_hnsw_ace.py b/python/cuvs/cuvs/tests/test_hnsw_ace.py index 663640e50d..183d530e7c 100644 --- a/python/cuvs/cuvs/tests/test_hnsw_ace.py +++ b/python/cuvs/cuvs/tests/test_hnsw_ace.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # From 8cd41911c4450a912ecb53ed31d901594962ea54 Mon Sep 17 00:00:00 2001 From: aamijar Date: Mon, 10 Aug 2026 22:04:22 +0000 Subject: [PATCH 23/24] revert another spdx change --- cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu b/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu index 16fa93f47b..adeb774a8b 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_graph_smaller_than_dataset.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ From 6ad9234655a846385ea8b839e5f569335fe95f6d Mon Sep 17 00:00:00 2001 From: aamijar Date: Mon, 10 Aug 2026 22:12:59 +0000 Subject: [PATCH 24/24] revert test to minimize diff --- cpp/tests/neighbors/ann_cagra.cuh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/tests/neighbors/ann_cagra.cuh b/cpp/tests/neighbors/ann_cagra.cuh index f85e36a6ef..1e969ec5fe 100644 --- a/cpp/tests/neighbors/ann_cagra.cuh +++ b/cpp/tests/neighbors/ann_cagra.cuh @@ -1627,12 +1627,12 @@ inline std::vector generate_inputs() inputs2 = raft::util::itertools::product( {100}, {1000}, - {1, 3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 768, 1024}, // dim - {16}, // k - {32}, // degree + {1, 3, 5, 7, 8, 17, 64, 128, 137, 192, 256, 512, 1024}, // dim + {16}, // k + {32}, // degree {graph_build_algo::IVF_PQ, graph_build_algo::NN_DESCENT, - graph_build_algo::ITERATIVE_CAGRA_SEARCH}, // Iterative cagra q build + graph_build_algo::ITERATIVE_CAGRA_SEARCH}, {search_algo::AUTO}, {10}, {0},