diff --git a/c/src/neighbors/cagra.cpp b/c/src/neighbors/cagra.cpp index 99e456e23c..ec3854465d 100644 --- a/c/src/neighbors/cagra.cpp +++ b/c/src/neighbors/cagra.cpp @@ -580,7 +580,7 @@ static void update_device_dataset_same_layout(raft::resources* res_ptr, auto* idx = reinterpret_cast*>(box->index_ptr); RAFT_EXPECTS(idx != nullptr, "cuvsCagraUpdateDataset: null index handle"); - idx->update_device_dataset_same_layout(*res_ptr, dataset_view); + idx->update_dataset(*res_ptr, dataset_view); }); } else if (box->layout == sg_cagra_c_api_index_box::dataset_layout::device_standard) { RAFT_EXPECTS(device_dataset->mem_type == CUVS_DATASET_MEM_TYPE_DEVICE && @@ -594,12 +594,12 @@ static void update_device_dataset_same_layout(raft::resources* res_ptr, auto* idx = reinterpret_cast*>(box->index_ptr); RAFT_EXPECTS(idx != nullptr, "cuvsCagraUpdateDataset: null index handle"); - idx->update_device_dataset_same_layout(*res_ptr, dataset_view); + idx->update_dataset(*res_ptr, dataset_view); }); } else { RAFT_FAIL( "cuvsCagraUpdateDataset: C++ " - "update_device_dataset_same_layout " + "update_dataset " "requires a device index and dataset"); } } @@ -711,7 +711,7 @@ void _from_args(cuvsResources_t res, auto dataset_view = cuvs::neighbors::make_device_padded_dataset_view(*res_ptr, mds); auto* raw = new cuvs::neighbors::cagra::device_padded_index( *res_ptr, metric); - raw->update_device_dataset_same_layout(*res_ptr, dataset_view); + raw->update_dataset(*res_ptr, dataset_view); update_graph_from_dlpack(raw); wrap_CPP_index_in_lifetime_holder_and_bind_to_C_index< T, @@ -721,7 +721,7 @@ void _from_args(cuvsResources_t res, auto dataset_view = cuvs::neighbors::make_device_standard_dataset_view(mds); auto* raw = new cuvs::neighbors::cagra::device_standard_index( *res_ptr, metric); - raw->update_device_dataset_same_layout(*res_ptr, dataset_view); + raw->update_dataset(*res_ptr, dataset_view); update_graph_from_dlpack(raw); wrap_CPP_index_in_lifetime_holder_and_bind_to_C_index< T, diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h index 6b275068b0..50c8ef580c 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h @@ -283,13 +283,13 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) *input_dataset_v_ = raft::make_device_matrix_view( mds.data_handle(), static_cast(nrow), static_cast(dim_)); auto index = cuvs::neighbors::cagra::build(handle_, params, pdv); - index.update_device_dataset_same_layout(handle_, pdv); + index.update_dataset(handle_, pdv); index_ = std::make_shared(std::move(index)); } else { auto padded = cuvs::neighbors::make_device_padded_dataset(handle_, mds); auto view = padded->as_dataset_view(); auto index = cuvs::neighbors::cagra::build(handle_, params, view); - index.update_device_dataset_same_layout(handle_, view); + index.update_dataset(handle_, view); *dataset_ = std::move(padded->data_); index_ = std::make_shared(std::move(index)); } @@ -318,17 +318,17 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) raft::resource::get_cuda_stream(handle_)); cuvs::neighbors::device_padded_dataset_view dv( raft::make_const_mdspan(sub_dataset_buffers_->back().view()), dim_); - sub_index.update_device_dataset_same_layout(handle_, dv); + sub_index.update_dataset(handle_, dv); } else { if (cuvs::neighbors::matrix_row_width_matches_cagra_required(sub_dev)) { auto pdv = cuvs::neighbors::make_device_padded_dataset_view(handle_, sub_dev); - sub_index.update_device_dataset_same_layout(handle_, pdv); + sub_index.update_dataset(handle_, pdv); } else { auto padded = cuvs::neighbors::make_device_padded_dataset(handle_, sub_dev); sub_dataset_buffers_->push_back(std::move(padded->data_)); cuvs::neighbors::device_padded_dataset_view pdv( raft::make_const_mdspan(sub_dataset_buffers_->back().view()), dim_); - sub_index.update_device_dataset_same_layout(handle_, pdv); + sub_index.update_dataset(handle_, pdv); } } } @@ -370,12 +370,12 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) if (sub_device && src_sub == req_sub) { auto pdv_sub = cuvs::neighbors::make_device_padded_dataset_view(handle_, mds_sub); sub_index = cuvs::neighbors::cagra::build(handle_, params, pdv_sub); - sub_index.update_device_dataset_same_layout(handle_, pdv_sub); + sub_index.update_dataset(handle_, pdv_sub); } else { auto padded_sub = cuvs::neighbors::make_device_padded_dataset(handle_, mds_sub); auto view = padded_sub->as_dataset_view(); auto index = cuvs::neighbors::cagra::build(handle_, params, view); - index.update_device_dataset_same_layout(handle_, view); + index.update_dataset(handle_, view); sub_dataset_buffers_->push_back(std::move(padded_sub->data_)); sub_index = std::move(index); } @@ -391,12 +391,12 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) if (sub_device && src_sub == req_sub) { auto pdv_sub = cuvs::neighbors::make_device_padded_dataset_view(handle_, mds_sub); sub_index = cuvs::neighbors::cagra::build(handle_, params, pdv_sub); - sub_index.update_device_dataset_same_layout(handle_, pdv_sub); + sub_index.update_dataset(handle_, pdv_sub); } else { auto padded_sub = cuvs::neighbors::make_device_padded_dataset(handle_, mds_sub); auto view = padded_sub->as_dataset_view(); auto index = cuvs::neighbors::cagra::build(handle_, params, view); - index.update_device_dataset_same_layout(handle_, view); + index.update_dataset(handle_, view); sub_dataset_buffers_->push_back(std::move(padded_sub->data_)); sub_index = std::move(index); } @@ -484,7 +484,7 @@ void cuvs_cagra::set_search_param(const search_param_base& param, *dataset_ = raft::make_device_matrix(handle_, 0, 0); cuvs::neighbors::device_padded_dataset_view empty_dv( raft::make_device_matrix_view(static_cast(nullptr), 0, this->dim_), this->dim_); - index_->update_device_dataset_same_layout(handle_, empty_dv); + index_->update_dataset(handle_, empty_dv); // Allocate space using the correct memory resource. RAFT_LOG_DEBUG("moving dataset to new memory space: %s", @@ -497,7 +497,7 @@ void cuvs_cagra::set_search_param(const search_param_base& param, raft::make_device_matrix_view( dataset_->data_handle(), dataset_->extent(0), dataset_->extent(1)), this->dim_); - index_->update_device_dataset_same_layout(handle_, dv); + index_->update_dataset(handle_, dv); need_dataset_update_ = false; needs_dynamic_batcher_update = true; @@ -554,17 +554,17 @@ void cuvs_cagra::set_search_dataset(const T* dataset, size_t nrow) raft::resource::get_cuda_stream(handle_)); cuvs::neighbors::device_padded_dataset_view dv( raft::make_const_mdspan(sub_dataset_buffers_->back().view()), dim_); - sub_index->update_device_dataset_same_layout(handle_, dv); + sub_index->update_dataset(handle_, dv); } else { if (cuvs::neighbors::matrix_row_width_matches_cagra_required(sub_dev)) { auto pdv = cuvs::neighbors::make_device_padded_dataset_view(handle_, sub_dev); - sub_index->update_device_dataset_same_layout(handle_, pdv); + sub_index->update_dataset(handle_, pdv); } else { auto padded = cuvs::neighbors::make_device_padded_dataset(handle_, sub_dev); sub_dataset_buffers_->push_back(std::move(padded->data_)); cuvs::neighbors::device_padded_dataset_view pdv( raft::make_const_mdspan(sub_dataset_buffers_->back().view()), dim_); - sub_index->update_device_dataset_same_layout(handle_, pdv); + sub_index->update_dataset(handle_, pdv); } } } diff --git a/cpp/cmake/patches/faiss-1.14-cuvs-26.08.diff b/cpp/cmake/patches/faiss-1.14-cuvs-26.08.diff index b36a18587a..9b81a4a2b3 100644 --- a/cpp/cmake/patches/faiss-1.14-cuvs-26.08.diff +++ b/cpp/cmake/patches/faiss-1.14-cuvs-26.08.diff @@ -79,7 +79,7 @@ index b331fdc..c7b5733 100644 - cuvs_index->update_dataset(raft_handle, dataset); + auto dataset_view = + cuvs::neighbors::make_device_padded_dataset_view(raft_handle, dataset); -+ cuvs_index->update_device_dataset_same_layout(raft_handle, dataset_view); ++ cuvs_index->update_dataset(raft_handle, dataset_view); } else { - auto dataset = raft::make_host_matrix_view( + auto host_dataset = raft::make_host_matrix_view( @@ -87,7 +87,7 @@ index b331fdc..c7b5733 100644 - cuvs_index->update_dataset(raft_handle, dataset); + host_to_device_dataset_ = + cuvs::neighbors::make_device_padded_dataset(raft_handle, host_dataset); -+ cuvs_index->update_device_dataset_same_layout(raft_handle, ++ cuvs_index->update_dataset(raft_handle, + host_to_device_dataset_->as_dataset_view()); } store_dataset_ = true; @@ -210,7 +210,7 @@ index 755817f..0eb03ae 100644 - cuvs_index->update_dataset(raft_handle, dataset); + auto dataset_view = + cuvs::neighbors::make_device_padded_dataset_view(raft_handle, dataset); -+ cuvs_index->update_device_dataset_same_layout(raft_handle, dataset_view); ++ cuvs_index->update_dataset(raft_handle, dataset_view); } else { - auto dataset = raft::make_host_matrix_view( + auto host_dataset = raft::make_host_matrix_view( @@ -218,7 +218,7 @@ index 755817f..0eb03ae 100644 - cuvs_index->update_dataset(raft_handle, dataset); + host_to_device_dataset_ = + cuvs::neighbors::make_device_padded_dataset(raft_handle, host_dataset); -+ cuvs_index->update_device_dataset_same_layout(raft_handle, ++ cuvs_index->update_dataset(raft_handle, + host_to_device_dataset_->as_dataset_view()); } store_dataset_ = true; diff --git a/cpp/include/cuvs/neighbors/cagra.hpp b/cpp/include/cuvs/neighbors/cagra.hpp index 43ae7a6235..2d0fca65c7 100644 --- a/cpp/include/cuvs/neighbors/cagra.hpp +++ b/cpp/include/cuvs/neighbors/cagra.hpp @@ -5,35 +5,28 @@ #pragma once -#include "common.hpp" #include +#include #include #include #include #include #include + #include #include -#include #include #include #include #include -#include -#include - -#include -#include -#include #include #include -#include -#include +#include #include -#include #include #include +#include #include #include @@ -203,7 +196,7 @@ struct index_params : cuvs::neighbors::index_params { * as the index is used. A device-backed index is ready to search immediately; a host-backed index * retains the dataset for operations such as serialization but is not searchable. * - `false` means `build` only builds the graph and the caller is expected to attach a dataset - * separately via `cuvs::neighbors::cagra::index::update_device_dataset_same_layout` before + * separately via `cuvs::neighbors::cagra::index::update_dataset` before * searching. * * Unlike the legacy behavior, no copy of the dataset is made: the index always stores a view. @@ -222,7 +215,7 @@ struct index_params : cuvs::neighbors::index_params { * auto index = cagra::build(res, index_params, dataset->as_dataset_view()); * // ASSERT(index.size() == 0); // no dataset yet * // Attach with a view (storage owned by `dataset`). - * index.update_device_dataset_same_layout(res, dataset->as_dataset_view()); + * index.update_dataset(res, dataset->as_dataset_view()); * cagra::search(res, search_params, index, queries, neighbors, distances); * @endcode */ @@ -439,12 +432,6 @@ struct extend_params { static_assert(std::is_aggregate_v); static_assert(std::is_aggregate_v); -template > -struct index; - /** * @defgroup cagra_cpp_index CAGRA index type * @{ @@ -461,7 +448,10 @@ struct index; * @tparam DatasetViewT concrete non-owning dataset view type stored by the index * */ -template +template > struct CUVS_EXPORT index : cuvs::neighbors::index { using index_params_type = cagra::index_params; using search_params_type = cagra::search_params; @@ -558,8 +548,8 @@ struct CUVS_EXPORT index : cuvs::neighbors::index { /** \endcond */ /** Construct a graph-only index with a zero-row dataset view placeholder. */ - index(raft::resources const& res, - cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded) + explicit index(raft::resources const& res, + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded) requires(cuvs::neighbors::ann_dataset_view) : cuvs::neighbors::index(), metric_(metric), @@ -623,19 +613,35 @@ struct CUVS_EXPORT index : cuvs::neighbors::index { raft::resource::sync_stream(res); } + /* Construct an index with a new dataset type by moving the old index and passing in a new + * dataset*/ + template + index(raft::resources const& res, index&& other, DatasetViewT dataset) + : metric_(other.metric_), + graph_(std::move(other.graph_)), + graph_view_(other.graph_view_), + source_indices_(std::move(other.source_indices_)), + n_rows_(other.n_rows_), + dim_(other.dim_), + graph_degree_(other.graph_degree_) + { + update_dataset(res, dataset); + } + /** * Replace the dataset with a new `dataset_view`. * * The index stores a copy of the view handle only (not the vector storage). The caller must * keep the underlying device data alive. Clears precomputed norms. */ - void update_device_dataset_same_layout(raft::resources const& res, DatasetViewT const& dataset) - requires cuvs::neighbors::is_device_dataset_view_v + void update_dataset(raft::resources const& res, DatasetViewT dataset) { dataset_ = dataset; dataset_norms_.reset(); - if (metric() == cuvs::distance::DistanceType::CosineExpanded) { - if (dataset_.n_rows() > 0) { compute_dataset_norms_(res); } + if constexpr (is_device_dataset_view_v) { + if (metric() == cuvs::distance::DistanceType::CosineExpanded) { + if (dataset_.n_rows() > 0) { compute_dataset_norms_(res); } + } } } @@ -849,6 +855,9 @@ struct CUVS_EXPORT index : cuvs::neighbors::index { } private: + template + friend struct index; + friend struct detail::fd_transfer; [[nodiscard]] inline auto steal_dataset_fd_() noexcept @@ -3590,6 +3599,36 @@ auto attach_dataset( cuvs::neighbors::device_padded_dataset_view const& padded_dataset) -> cuvs::neighbors::mg_index, uint8_t, uint32_t>; +/** + * @brief Consume a standard MG CAGRA index and attach a padded dataset for search. + * + * This moves each rank-local CAGRA graph into the returned padded MG index. + */ +auto update_dataset( + const raft::resources& clique, + cuvs::neighbors::mg_index, float, uint32_t>&& idx, + cuvs::neighbors::device_padded_dataset_view const& padded_dataset) + -> cuvs::neighbors::mg_index, float, uint32_t>; + +auto update_dataset( + const raft::resources& clique, + cuvs::neighbors::mg_index, half, uint32_t>&& idx, + cuvs::neighbors::device_padded_dataset_view const& padded_dataset) + -> cuvs::neighbors::mg_index, half, uint32_t>; + +auto update_dataset( + const raft::resources& clique, + cuvs::neighbors::mg_index, int8_t, uint32_t>&& idx, + cuvs::neighbors::device_padded_dataset_view const& padded_dataset) + -> cuvs::neighbors::mg_index, int8_t, uint32_t>; + +auto update_dataset( + const raft::resources& clique, + cuvs::neighbors::mg_index, uint8_t, uint32_t>&& + idx, + cuvs::neighbors::device_padded_dataset_view const& padded_dataset) + -> cuvs::neighbors::mg_index, uint8_t, uint32_t>; + /** * @brief Update an existing padded MG CAGRA index with a padded dataset of the same layout. */ @@ -4544,7 +4583,7 @@ auto convert_standard_to_padded_index( if (standard_idx.source_indices().has_value()) { out.update_source_indices(res, standard_idx.source_indices().value()); } - out.update_device_dataset_same_layout(res, padded_dataset); + out.update_dataset(res, padded_dataset); return out; } @@ -4556,8 +4595,8 @@ auto convert_standard_to_padded_index( * For host/standard index layouts, this function converts to and returns a new * `device_padded_index`. * - * If `idx` is already a `device_padded_index`, call `idx.update_device_dataset_same_layout(res, - * device_padded_dataset)` directly to avoid an unnecessary copy path. + * If `idx` is already a `device_padded_index`, call + * `idx.update_dataset(res, device_padded_dataset)` directly to avoid an unnecessary copy path. * * @param[in] res RAFT resources * @param[in] idx CAGRA index in any host/device + standard/padded layout @@ -4580,7 +4619,7 @@ auto attach_dataset( return convert_standard_to_padded_index(res, dev_std, device_padded_dataset); } else if constexpr (cuvs::neighbors::is_host_padded_dataset_view_v) { auto dev_pad = detail::convert_host_to_device_index(res, idx); - dev_pad.update_device_dataset_same_layout(res, device_padded_dataset); + dev_pad.update_dataset(res, device_padded_dataset); return dev_pad; } else if constexpr (cuvs::neighbors::is_device_standard_dataset_view_v) { return convert_standard_to_padded_index(res, idx, device_padded_dataset); @@ -4588,16 +4627,93 @@ auto attach_dataset( RAFT_LOG_WARN( "cagra::attach_dataset called with an already device-padded index. " "To avoid an unnecessary index copy, call " - "index.update_device_dataset_same_layout(res, device_padded_dataset) " + "index.update_dataset(res, device_padded_dataset) " "directly on the original index."); RAFT_FAIL( "cagra::attach_dataset: device_padded_index input is not supported in this overload. " - "Call index.update_device_dataset_same_layout(res, device_padded_dataset) directly."); + "Call index.update_dataset(res, device_padded_dataset) directly."); } else { static_assert(!sizeof(IndexViewT), "Unsupported CAGRA index dataset view type"); } } +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset(raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_standard_dataset_view dataset) + -> index>; +auto update_dataset(raft::resources const& res, + index>&& cagra_index, + device_standard_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_standard_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_standard_dataset_view dataset) + -> index>; +auto update_dataset(raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset(raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; + } // namespace cagra } // namespace neighbors } // namespace CUVS_EXPORT cuvs diff --git a/cpp/include/cuvs/preprocessing/quantize/pq.hpp b/cpp/include/cuvs/preprocessing/quantize/pq.hpp index 29d0bbb1e3..e6ffc78f53 100644 --- a/cpp/include/cuvs/preprocessing/quantize/pq.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/pq.hpp @@ -268,7 +268,7 @@ template * Typical **CAGRA** usage: build the graph on dense vectors, then attach VPQ for search (metric * must remain `L2Expanded` for this path). Train VPQ from the same CAGRA-padded device layout you * used for graph build, keep the `device_vpq_dataset` alive, and call - * `index::update_device_dataset_same_layout` with a non-owning view. + * `index::update_dataset` with a non-owning view. * * @code{.cpp} * #include @@ -278,7 +278,7 @@ template * // `padded` is a `device_padded_dataset_view` view of those same rows. * cuvs::neighbors::vpq_params vpq_params{}; * auto vpq = cuvs::preprocessing::quantize::pq::make_vpq_dataset(res, vpq_params, padded.view()); - * idx.update_device_dataset_same_layout(res, vpq.as_dataset_view()); + * idx.update_dataset(res, vpq.as_dataset_view()); * @endcode */ template diff --git a/cpp/src/neighbors/cagra.cuh b/cpp/src/neighbors/cagra.cuh index 80e2f2a07e..215763b9ff 100644 --- a/cpp/src/neighbors/cagra.cuh +++ b/cpp/src/neighbors/cagra.cuh @@ -306,7 +306,7 @@ auto build(raft::resources const& res, const index_params& params, DatasetViewT } else if constexpr (cuvs::neighbors::is_dense_row_major_device_dataset_view_v) { auto idx = cuvs::neighbors::cagra::detail::build_from_device_matrix( res, params, dataset); - if (params.attach_dataset_on_build) { idx.update_device_dataset_same_layout(res, dataset); } + if (params.attach_dataset_on_build) { idx.update_dataset(res, dataset); } return idx; } else { if (std::holds_alternative(params.graph_build_params)) { @@ -587,6 +587,21 @@ void search( } } +template +auto update_dataset(raft::resources const& res, + index&& cagra_index, + DstDatasetViewT dataset) -> index +{ + static_assert(!std::is_same_v, + "For updating a dataset of the same type use the cagra_index.update_dataset() " + "function instead."); + index new_index(res, std::move(cagra_index), dataset); + return new_index; +} + /** @} */ // end group cagra } // namespace cuvs::neighbors::cagra diff --git a/cpp/src/neighbors/cagra_build_inst.cu.in b/cpp/src/neighbors/cagra_build_inst.cu.in index acaaa942c1..49ff8a18ce 100644 --- a/cpp/src/neighbors/cagra_build_inst.cu.in +++ b/cpp/src/neighbors/cagra_build_inst.cu.in @@ -9,6 +9,7 @@ #include #include +#include namespace { @@ -57,4 +58,29 @@ CUVS_DEFINE_CAGRA_BUILD_OVERLOAD(inst_host_standard_view_t, #undef CUVS_DEFINE_CAGRA_BUILD_OVERLOAD +#define CUVS_INST_CAGRA_UPDATE_DATASET(T, IdxT, SrcDatasetViewT, DstDatasetViewT) \ + auto update_dataset(raft::resources const& res, \ + index&& cagra_index, \ + DstDatasetViewT dataset) -> index \ + { \ + return cuvs::neighbors::cagra::update_dataset( \ + res, std::move(cagra_index), dataset); \ + } + +CUVS_INST_CAGRA_UPDATE_DATASET(data_t, + index_t, + inst_host_standard_view_t, + inst_device_padded_view_t); +CUVS_INST_CAGRA_UPDATE_DATASET(data_t, + index_t, + inst_host_standard_view_t, + inst_device_standard_view_t); +CUVS_INST_CAGRA_UPDATE_DATASET(data_t, index_t, inst_host_padded_view_t, inst_device_padded_view_t); +CUVS_INST_CAGRA_UPDATE_DATASET(data_t, + index_t, + inst_device_standard_view_t, + inst_device_padded_view_t); + +#undef CUVS_INST_CAGRA_UPDATE_DATASET + } // namespace cuvs::neighbors::cagra diff --git a/cpp/src/neighbors/detail/cagra/add_nodes.cuh b/cpp/src/neighbors/detail/cagra/add_nodes.cuh index 62a77239b2..14108f8410 100644 --- a/cpp/src/neighbors/detail/cagra/add_nodes.cuh +++ b/cpp/src/neighbors/detail/cagra/add_nodes.cuh @@ -325,7 +325,7 @@ void add_graph_nodes( // add_node_core() uses CAGRA search internally, which requires a padded device dataset. // Keep this path allocation-free by requiring pre-padded chunk views. auto pdv = cuvs::neighbors::make_device_padded_dataset_view(handle, dataset_view); - internal_index.update_device_dataset_same_layout(handle, pdv); + internal_index.update_dataset(handle, pdv); // Note: The graph is copied to the device memory. internal_index.update_graph(handle, graph_view); @@ -388,7 +388,7 @@ void extend_core(raft::resources const& handle, if constexpr (cuvs::neighbors::is_empty_dataset_view_v>) { RAFT_FAIL( "cagra::extend only supports an index to which the dataset is attached. Please check if the " - "index has an empty dataset; attach one with update_device_dataset_same_layout before " + "index has an empty dataset; attach one with update_dataset before " "extend."); } else if constexpr (!cuvs::neighbors::is_padded_dataset_view_v>) { RAFT_FAIL("cagra::extend only supports an uncompressed padded dataset index"); @@ -408,7 +408,7 @@ void extend_core(raft::resources const& handle, cuvs::neighbors::cagra::add_graph_nodes( handle, extended_strided, index, updated_graph.view(), params); - index.update_device_dataset_same_layout(handle, extended_dataset); + index.update_dataset(handle, extended_dataset); index.update_graph(handle, raft::make_const_mdspan(updated_graph.view())); } } diff --git a/cpp/src/neighbors/detail/cagra/cagra_build.cuh b/cpp/src/neighbors/detail/cagra/cagra_build.cuh index c06f9b12e3..d371d213e3 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_build.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_build.cuh @@ -2390,7 +2390,7 @@ auto build_from_host_matrix(raft::resources const& res, * Build from a dense device `dataset_view` (padded or standard). VPQ views are rejected by * `cagra::build()` before this entry point is reached. Also used from ACE sub-builds and merge. * The returned index contains only the optimized graph; call - * `index::update_device_dataset_same_layout` before search. + * `index::update_dataset` before search. */ template requires cuvs::neighbors::is_dense_row_major_device_dataset_view_v diff --git a/cpp/src/neighbors/detail/cagra/cagra_merge.cuh b/cpp/src/neighbors/detail/cagra/cagra_merge.cuh index 3fd1963268..8dca813689 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_merge.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_merge.cuh @@ -80,7 +80,7 @@ cuvs::neighbors::cagra::index merge( RAFT_EXPECTS( dataset.n_rows() != 0, "cagra::merge only supports an index to which the dataset is attached. Please check if " - "the index has an empty dataset; attach one with update_device_dataset_same_layout " + "the index has an empty dataset; attach one with update_dataset " "before merge."); if (dim == 0) { dim = index->dim(); @@ -174,7 +174,7 @@ cuvs::neighbors::cagra::index merge( auto index = ::cuvs::neighbors::cagra::detail::build_from_device_matrix( handle, params, merged_dataset); - index.update_device_dataset_same_layout(handle, merged_dataset); + index.update_dataset(handle, merged_dataset); RAFT_LOG_DEBUG("cagra merge: using device memory for merged dataset"); return index; } @@ -187,7 +187,7 @@ cuvs::neighbors::cagra::index merge( merge_dataset(output_view.data_handle(), static_cast(stride)); auto index = ::cuvs::neighbors::cagra::detail::build_from_device_matrix( handle, params, merged_dataset); - index.update_device_dataset_same_layout(handle, merged_dataset); + index.update_dataset(handle, merged_dataset); RAFT_LOG_DEBUG("cagra merge: using device memory for merged dataset"); return index; } diff --git a/cpp/src/neighbors/detail/cagra/cagra_search.cuh b/cpp/src/neighbors/detail/cagra/cagra_search.cuh index 165e478337..a5562c193c 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_search.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_search.cuh @@ -233,7 +233,7 @@ void search_main(raft::resources const& res, if constexpr (cuvs::neighbors::is_empty_dataset_view_v) { RAFT_FAIL( "Attempted to search without a dataset. Please call " - "index.update_device_dataset_same_layout(...) first."); + "index.update_dataset(...) first."); } else if constexpr (cuvs::neighbors::is_device_vpq_f32_dataset_view_v) { RAFT_FAIL("FP32 VPQ dataset support is coming soon"); } else if constexpr (cuvs::neighbors::is_device_vpq_f16_dataset_view_v) { diff --git a/cpp/src/neighbors/detail/tiered_index.cuh b/cpp/src/neighbors/detail/tiered_index.cuh index 368a41ee44..6abd57507f 100644 --- a/cpp/src/neighbors/detail/tiered_index.cuh +++ b/cpp/src/neighbors/detail/tiered_index.cuh @@ -126,8 +126,7 @@ struct index_state { { auto index = std::forward(build_fn)(res, tiered_params, dataset); if constexpr (std::is_same_v>) { - index.update_device_dataset_same_layout( - res, cuvs::neighbors::make_device_standard_dataset_view(dataset)); + index.update_dataset(res, cuvs::neighbors::make_device_standard_dataset_view(dataset)); } return std::make_shared(std::move(index)); } @@ -299,8 +298,7 @@ inline void update_cagra_ann_dataset_for_stride( cuvs::neighbors::cagra::device_standard_index& ann_index, raft::device_matrix_view dataset) { - ann_index.update_device_dataset_same_layout( - res, cuvs::neighbors::make_device_standard_dataset_view(dataset)); + ann_index.update_dataset(res, cuvs::neighbors::make_device_standard_dataset_view(dataset)); } /** diff --git a/cpp/src/neighbors/iface/iface.hpp b/cpp/src/neighbors/iface/iface.hpp index 5d5ee76406..fb9905737a 100644 --- a/cpp/src/neighbors/iface/iface.hpp +++ b/cpp/src/neighbors/iface/iface.hpp @@ -57,12 +57,12 @@ void cagra_build_from_device_dataset( if constexpr (std::is_same_v>) { auto padded = cuvs::neighbors::make_device_padded_dataset_view(h, dview); auto index = cuvs::neighbors::cagra::build(h, cagra_params, padded); - index.update_device_dataset_same_layout(h, padded); + index.update_dataset(h, padded); interface.index_.emplace(std::move(index)); } else { auto standard = cuvs::neighbors::make_device_standard_dataset_view(dview); auto index = cuvs::neighbors::cagra::build(h, cagra_params, standard); - index.update_device_dataset_same_layout(h, standard); + index.update_dataset(h, standard); interface.index_.emplace(std::move(index)); } interface.cagra_owned_standard_dataset_.reset(); @@ -106,10 +106,10 @@ void build(const raft::resources& handle, if constexpr (std::is_same>::value) { cuvs::neighbors::host_padded_dataset_view host_padded( host_view, static_cast(host_view.extent(1))); - auto host_idx = cuvs::neighbors::cagra::build(handle, cagra_params, host_padded); - auto padded_r = cuvs::neighbors::make_device_padded_dataset(handle, index_dataset); - auto device_idx = - cuvs::neighbors::cagra::attach_dataset(handle, host_idx, padded_r->as_dataset_view()); + auto host_idx = cuvs::neighbors::cagra::build(handle, cagra_params, host_padded); + auto padded_r = cuvs::neighbors::make_device_padded_dataset(handle, index_dataset); + auto device_idx = cuvs::neighbors::cagra::update_dataset( + handle, std::move(host_idx), padded_r->as_dataset_view()); interface.cagra_owned_padded_dataset_ = std::move(padded_r); interface.cagra_owned_standard_dataset_.reset(); interface.index_.emplace(std::move(device_idx)); @@ -121,9 +121,8 @@ void build(const raft::resources& handle, index_dataset, static_cast(index_dataset.extent(1)), static_cast(index_dataset.stride(0))); - auto device_idx = - cuvs::neighbors::cagra::detail::convert_host_to_device_index(handle, host_idx); - device_idx.update_device_dataset_same_layout(handle, standard_r->as_dataset_view()); + auto device_idx = cuvs::neighbors::cagra::update_dataset( + handle, std::move(host_idx), standard_r->as_dataset_view()); interface.cagra_owned_standard_dataset_ = std::move(standard_r); interface.cagra_owned_padded_dataset_.reset(); interface.index_.emplace(std::move(device_idx)); diff --git a/cpp/src/neighbors/mg/mg_cagra_inst.cu.in b/cpp/src/neighbors/mg/mg_cagra_inst.cu.in index 78d6b109ef..4b0f8de64f 100644 --- a/cpp/src/neighbors/mg/mg_cagra_inst.cu.in +++ b/cpp/src/neighbors/mg/mg_cagra_inst.cu.in @@ -101,6 +101,27 @@ void distribute_padded_dataset( return out; \ } \ \ + auto update_dataset( \ + const raft::resources& res, \ + cuvs::neighbors::mg_index, T, IdxT>&& idx, \ + cuvs::neighbors::device_padded_dataset_view const& padded_dataset) \ + -> cuvs::neighbors::mg_index, T, IdxT> \ + { \ + cuvs::neighbors::mg_index, T, IdxT> out(res, idx.mode_); \ + out.ann_interfaces_.resize(idx.num_ranks_); \ + detail::distribute_padded_dataset( \ + res, idx, padded_dataset, [&](const raft::resources& dev_res, int rank, auto dataset) { \ + auto& in_if = idx.ann_interfaces_[rank]; \ + auto& out_if = out.ann_interfaces_[rank]; \ + auto padded_idx = cuvs::neighbors::cagra::update_dataset( \ + dev_res, std::move(in_if.index_.value()), dataset->as_dataset_view()); \ + out_if.cagra_owned_padded_dataset_ = std::move(dataset); \ + out_if.cagra_owned_standard_dataset_.reset(); \ + out_if.index_.emplace(std::move(padded_idx)); \ + }); \ + return out; \ + } \ + \ void update_device_dataset_same_layout( \ const raft::resources& res, \ cuvs::neighbors::mg_index, T, IdxT>& idx, \ @@ -110,7 +131,7 @@ void distribute_padded_dataset( res, idx, padded_dataset, [&](const raft::resources& dev_res, int rank, auto dataset) { \ auto& ann_if = idx.ann_interfaces_[rank]; \ auto& padded_idx = ann_if.index_.value(); \ - padded_idx.update_device_dataset_same_layout(dev_res, dataset->as_dataset_view()); \ + padded_idx.update_dataset(dev_res, dataset->as_dataset_view()); \ ann_if.cagra_owned_padded_dataset_ = std::move(dataset); \ ann_if.cagra_owned_standard_dataset_.reset(); \ }); \ diff --git a/cpp/src/neighbors/tiered_index.cu b/cpp/src/neighbors/tiered_index.cu index f5942001d7..7d1e684492 100644 --- a/cpp/src/neighbors/tiered_index.cu +++ b/cpp/src/neighbors/tiered_index.cu @@ -149,15 +149,12 @@ void extend(raft::resources const& res, if (storage->num_rows_allocated != idx->state->storage->num_rows_allocated) { // CAGRA could be holding on to a non-owning view of the previous dataset in the ann_index, // which is problematic since the underlying ownership of the dataset could be freed here - // call cagra::index::update_device_dataset_same_layout on it to update the ann_index to point - // to the - // new dataset + // call cagra::index::update_dataset on it to update the ann_index to point to the new dataset if (next_state->ann_index) { auto dataset = raft::make_device_matrix_view( storage->dataset.data(), next_state->ann_rows(), storage->dim); - // Block 'search' calls during the update_device_dataset_same_layout call to ensure that this - // doesn't cause issues in a multithreaded environment + // Block 'search' calls during update_dataset to avoid issues in a multithreaded environment std::unique_lock lock(idx->ann_mutex); detail::update_cagra_ann_dataset_for_stride(res, *next_state->ann_index, dataset); } diff --git a/cpp/tests/neighbors/ann_cagra.cuh b/cpp/tests/neighbors/ann_cagra.cuh index 529b8fe038..32bae90d8e 100644 --- a/cpp/tests/neighbors/ann_cagra.cuh +++ b/cpp/tests/neighbors/ann_cagra.cuh @@ -76,7 +76,7 @@ void cagra_build_into_index( return; } index = cagra::build(res, params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); } struct test_cagra_sample_filter { @@ -489,9 +489,7 @@ class AnnCagraTest : public ::testing::TestWithParam { std::unique_ptr> loaded_dataset; cagra::deserialize(handle_, index_file.filename, &index, &loaded_dataset); - if (!ps.include_serialized_dataset) { - index.update_device_dataset_same_layout(handle_, device_padded.view); - } + if (!ps.include_serialized_dataset) { index.update_dataset(handle_, device_padded.view); } auto search_queries_view = raft::make_device_matrix_view( search_queries.data(), ps.n_queries, ps.dim); @@ -2126,7 +2124,7 @@ class AnnCagraMultiPartitionTest : public ::testing::TestWithParam padded_small( res, small_dataset_view); auto small_index = cagra::build(res, small_index_params, padded_small.view); - small_index.update_device_dataset_same_layout(res, padded_small.view); + small_index.update_dataset(res, padded_small.view); raft::resource::sync_stream(res); // Step 2: Update to FULL dataset (1000 points) but keep small graph (500 nodes) // This creates the exact bug scenario: dataset.size=1000, graph.extent(0)=500 - small_index.update_device_dataset_same_layout(res, - cuvs::neighbors::make_device_padded_dataset_view( - res, raft::make_const_mdspan(dataset.view()))); + small_index.update_dataset(res, + cuvs::neighbors::make_device_padded_dataset_view( + res, raft::make_const_mdspan(dataset.view()))); // Verify the mismatch - THIS IS THE BUG SCENARIO! ASSERT_EQ(small_index.graph().extent(0), n_graph); // Graph has 500 nodes diff --git a/cpp/tests/neighbors/ann_cagra/bug_issue_93_reproducer.cu b/cpp/tests/neighbors/ann_cagra/bug_issue_93_reproducer.cu index 9e2cde1d80..320626b211 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_issue_93_reproducer.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_issue_93_reproducer.cu @@ -75,7 +75,7 @@ TEST(Issue93Reproducer, ConcurrentSearchDifferentGraphDegrees) padded_builders.emplace_back(handle, raft::make_const_mdspan(database.view())); auto index = cagra::build(handle, ip, padded_builders.back().view); - index.update_device_dataset_same_layout(handle, padded_builders.back().view); + index.update_dataset(handle, padded_builders.back().view); indices.push_back(std::move(index)); } raft::resource::sync_stream(handle); diff --git a/cpp/tests/neighbors/ann_cagra/bug_iterative_cagra_build.cu b/cpp/tests/neighbors/ann_cagra/bug_iterative_cagra_build.cu index 3d6b5c98a2..c3774f5cfd 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_iterative_cagra_build.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_iterative_cagra_build.cu @@ -38,7 +38,7 @@ class CagraIterativeBuildBugTest : public ::testing::Test { cuvs::neighbors::test::padded_device_matrix_for_cagra padded( res, raft::make_const_mdspan(dataset->view())); auto cagra_index = cagra::build(res, index_params, padded.view); - cagra_index.update_device_dataset_same_layout(res, padded.view); + cagra_index.update_dataset(res, padded.view); raft::resource::sync_stream(res); // Verify the index was built successfully diff --git a/cpp/tests/neighbors/ann_cagra/bug_multi_cta_crash.cu b/cpp/tests/neighbors/ann_cagra/bug_multi_cta_crash.cu index 6e965bfcb9..85d8eb8315 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_multi_cta_crash.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_multi_cta_crash.cu @@ -30,7 +30,7 @@ class AnnCagraBugMultiCTACrash : public ::testing::TestWithParamview())); auto cagra_index = cagra::build(res, cagra_index_params, build_padded_->view); - cagra_index.update_device_dataset_same_layout(res, build_padded_->view); + cagra_index.update_dataset(res, build_padded_->view); raft::resource::sync_stream(res); cagra::search_params cagra_search_params; diff --git a/cpp/tests/neighbors/ann_cagra/test_float_uint32_t.cu b/cpp/tests/neighbors/ann_cagra/test_float_uint32_t.cu index de06bef34a..64fb5c91af 100644 --- a/cpp/tests/neighbors/ann_cagra/test_float_uint32_t.cu +++ b/cpp/tests/neighbors/ann_cagra/test_float_uint32_t.cu @@ -96,7 +96,7 @@ void expect_multi_partition_search_throws( part_padded.emplace_back(handle, view); auto const& padded = part_padded.back().view; part_indices.push_back(cagra::build(handle, index_params, padded)); - part_indices.back().update_device_dataset_same_layout(handle, padded); + part_indices.back().update_dataset(handle, padded); } std::vector*> index_ptrs; for (auto& idx : part_indices) { diff --git a/cpp/tests/neighbors/dynamic_batching/test_cagra.cu b/cpp/tests/neighbors/dynamic_batching/test_cagra.cu index 802bae4bce..336daab5f4 100644 --- a/cpp/tests/neighbors/dynamic_batching/test_cagra.cu +++ b/cpp/tests/neighbors/dynamic_batching/test_cagra.cu @@ -22,7 +22,7 @@ auto build_cagra_with_dataset(raft::resources const& res, { auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, dataset); auto index = cagra::build(res, params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); return index; } diff --git a/cpp/tests/neighbors/hnsw.cu b/cpp/tests/neighbors/hnsw.cu index d9fedf0ed7..01eeeb511c 100644 --- a/cpp/tests/neighbors/hnsw.cu +++ b/cpp/tests/neighbors/hnsw.cu @@ -98,7 +98,7 @@ class AnnHNSWTest : public ::testing::TestWithParam { cuvs::neighbors::test::padded_device_matrix_for_cagra padded(handle_, database_view); auto index = cuvs::neighbors::cagra::build(handle_, index_params, padded.view); - index.update_device_dataset_same_layout(handle_, padded.view); + index.update_dataset(handle_, padded.view); raft::resource::sync_stream(handle_); cuvs::neighbors::hnsw::search_params search_params; diff --git a/examples/cpp/src/cagra_bloom_filter_example.cu b/examples/cpp/src/cagra_bloom_filter_example.cu index 7d505dd4f0..9ce1ff6baa 100644 --- a/examples/cpp/src/cagra_bloom_filter_example.cu +++ b/examples/cpp/src/cagra_bloom_filter_example.cu @@ -72,7 +72,7 @@ int main() auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, raft::make_const_mdspan(dataset.view())); auto index = cuvs::neighbors::cagra::build(res, index_params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); // Build one global bloom filter over the index: bulk-insert every valid row id once. std::vector valid_ids_host; diff --git a/examples/cpp/src/cagra_example.cu b/examples/cpp/src/cagra_example.cu index bd35b4bd86..b7a96fa39a 100644 --- a/examples/cpp/src/cagra_example.cu +++ b/examples/cpp/src/cagra_example.cu @@ -34,7 +34,7 @@ void cagra_build_search_simple(raft::device_resources const& dev_resources, std::cout << "Building CAGRA index (search graph)" << std::endl; auto padded = cuvs::neighbors::make_device_padded_dataset_view(dev_resources, dataset); auto index = cagra::build(dev_resources, index_params, padded); - index.update_device_dataset_same_layout(dev_resources, padded); + index.update_dataset(dev_resources, padded); std::cout << "CAGRA index has " << index.size() << " vectors" << std::endl; std::cout << "CAGRA graph has degree " << index.graph_degree() << ", graph size [" diff --git a/examples/cpp/src/cagra_filter_udf_example.cu b/examples/cpp/src/cagra_filter_udf_example.cu index 56121ecb73..1b331719b0 100644 --- a/examples/cpp/src/cagra_filter_udf_example.cu +++ b/examples/cpp/src/cagra_filter_udf_example.cu @@ -147,7 +147,7 @@ int main() std::cout << "Building CAGRA index" << std::endl; auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, dataset.view()); auto index = cuvs::neighbors::cagra::build(res, index_params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); std::vector row_tenant_ids(n_rows); std::vector row_timestamps(n_rows); diff --git a/examples/cpp/src/cagra_persistent_example.cu b/examples/cpp/src/cagra_persistent_example.cu index 4580526e79..494a8230a3 100644 --- a/examples/cpp/src/cagra_persistent_example.cu +++ b/examples/cpp/src/cagra_persistent_example.cu @@ -71,7 +71,7 @@ void cagra_build_search_variants(raft::device_resources const& res, std::cout << "Building CAGRA index (search graph)" << std::endl; auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, dataset); auto index = cagra::build(res, index_params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); std::cout << "CAGRA index has " << index.size() << " vectors" << std::endl; std::cout << "CAGRA graph has degree " << index.graph_degree() << ", graph size [" diff --git a/examples/cpp/src/dynamic_batching_example.cu b/examples/cpp/src/dynamic_batching_example.cu index dc9a29e759..d52d59f49a 100644 --- a/examples/cpp/src/dynamic_batching_example.cu +++ b/examples/cpp/src/dynamic_batching_example.cu @@ -116,7 +116,7 @@ void dynamic_batching_example(raft::resources const& res, std::cout << "Building CAGRA index (search graph)" << std::endl; auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, dataset); auto orig_index = cagra::build(res, orig_index_params, padded); - orig_index.update_device_dataset_same_layout(res, padded); + orig_index.update_dataset(res, padded); std::cout << "CAGRA index has " << orig_index.size() << " vectors" << std::endl; std::cout << "CAGRA graph has degree " << orig_index.graph_degree() << ", graph size ["