From 3060ecdde61fb02c5ee5e213cfa0562563e6c9f4 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 21 May 2026 12:57:13 +0800 Subject: [PATCH 1/3] [improvement](be) Track inverted index writer memory estimate ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Add minimal memory-estimate instrumentation for inverted index writing so high-cardinality BKD index build peak memory can be measured. ### Release note None ### Check List (For Author) - Test: Unit Test - PATH=/mnt/disk6/common/ldb_toolchain_toucan/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk1/jiangkai/.bun/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk1/jiangkai/.codex/tmp/arg0/codex-arg0dYzs9D:/mnt/disk6/common/node-v24.14.1-linux-x64/lib/node_modules/@openai/codex/node_modules/@openai/codex-linux-x64/vendor/x86_64-unknown-linux-musl/path:/mnt/disk1/jiangkai/.bun/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk1/jiangkai/.bun/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/usr/share/Modules/bin:/usr/lib64/ccache:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin build-support/clang-format.sh - PATH=/mnt/disk6/common/ldb_toolchain_toucan/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk1/jiangkai/.bun/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk1/jiangkai/.codex/tmp/arg0/codex-arg0dYzs9D:/mnt/disk6/common/node-v24.14.1-linux-x64/lib/node_modules/@openai/codex/node_modules/@openai/codex-linux-x64/vendor/x86_64-unknown-linux-musl/path:/mnt/disk1/jiangkai/.bun/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk1/jiangkai/.bun/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/usr/share/Modules/bin:/usr/lib64/ccache:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin build-support/check-format.sh - sh run-be-ut.sh --run --filter=InvertedIndexWriterTest.HighCardinalityBkdMemoryEstimate - build-support/run-clang-tidy.sh --build-dir be/ut_build_ASAN (attempted; failed because clang-tidy could not analyze the touched files due pre-existing/environment diagnostics) - Behavior changed: No - Does this need documentation: No --- .../index/inverted/inverted_index_writer.cpp | 78 ++++++++++++++++++- be/src/storage/segment/column_writer.cpp | 21 ++++- .../segment/inverted_index_writer_test.cpp | 69 +++++++++++++++- 3 files changed, 161 insertions(+), 7 deletions(-) diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index 0f82b5225e666e..902c630fb84394 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -17,6 +17,9 @@ #include "storage/index/inverted/inverted_index_writer.h" +#include + +#include "common/cast_set.h" #include "storage/index/inverted/analyzer/analyzer.h" #include "storage/index/inverted/inverted_index_common.h" #include "storage/index/inverted/inverted_index_fs_directory.h" @@ -32,6 +35,71 @@ const int32_t MAX_LEAF_COUNT = 1024; const float MAXMBSortInHeap = 512.0 * 8; const int DIMS = 1; +namespace { + +template +int64_t vector_memory_size(const std::vector& vector) { + return cast_set(vector.capacity() * sizeof(T)); +} + +int64_t bytes_ref_memory_size(const std::shared_ptr& bytes_ref) { + if (bytes_ref == nullptr) { + return 0; + } + return cast_set(sizeof(lucene::util::BytesRef)) + vector_memory_size(bytes_ref->bytes); +} + +int64_t heap_point_writer_memory_size( + const std::shared_ptr& writer) { + if (writer == nullptr) { + return 0; + } + + int64_t size = cast_set(sizeof(lucene::util::bkd::heap_point_writer)); + size += vector_memory_size(writer->doc_IDs_); + size += vector_memory_size(writer->ords_long_); + size += vector_memory_size(writer->ords_); + size += vector_memory_size(writer->blocks_); + for (const auto& block : writer->blocks_) { + size += vector_memory_size(block); + } + size += bytes_ref_memory_size(writer->cache_); + size += bytes_ref_memory_size(writer->cache1_); + return size; +} + +int64_t bkd_writer_memory_size(const std::shared_ptr& writer) { + if (writer == nullptr) { + return 0; + } + + int64_t size = cast_set(sizeof(lucene::util::bkd::bkd_writer)); + size += vector_memory_size(writer->min_packed_value_); + size += vector_memory_size(writer->max_packed_value_); + size += vector_memory_size(writer->scratch_diff_); + size += vector_memory_size(writer->scratch1_); + size += vector_memory_size(writer->scratch2_); + size += vector_memory_size(writer->common_prefix_lengths_); + size += heap_point_writer_memory_size(writer->heap_point_writer_); + return size; +} + +int64_t ram_directory_memory_size(const std::shared_ptr& dir) { + if (dir == nullptr || std::strcmp(dir->getObjectName(), "DorisRAMFSDirectory") != 0) { + return 0; + } + + int64_t size = 0; + std::vector files; + dir->list(&files); + for (const auto& file : files) { + size += dir->fileLength(file.c_str()); + } + return size; +} + +} // namespace + template InvertedIndexColumnWriter::InvertedIndexColumnWriter(const std::string& field_name, IndexFileWriter* index_file_writer, @@ -566,8 +634,12 @@ Status InvertedIndexColumnWriter::add_value(const CppType& value) { template int64_t InvertedIndexColumnWriter::size() const { - //TODO: get memory size of inverted index - return 0; + int64_t size = cast_set(_null_bitmap.getSizeInBytes(false)); + if constexpr (field_is_numeric_type(field_type)) { + size += bkd_writer_memory_size(_bkd_writer); + } + size += ram_directory_memory_size(_dir); + return size; } template @@ -683,4 +755,4 @@ template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/src/storage/segment/column_writer.cpp b/be/src/storage/segment/column_writer.cpp index 15d4f558f9b054..542d0b597c0892 100644 --- a/be/src/storage/segment/column_writer.cpp +++ b/be/src/storage/segment/column_writer.cpp @@ -721,6 +721,12 @@ uint64_t ScalarColumnWriter::estimate_buffer_size() { if (_opts.need_bloom_filter) { size += _bloom_filter_index_builder->size(); } + if (_opts.need_inverted_index) { + for (const auto& builder : _inverted_index_builders) { + DORIS_CHECK(builder != nullptr); + size += builder->size(); + } + } return size; } @@ -1121,9 +1127,18 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { } uint64_t ArrayColumnWriter::estimate_buffer_size() { - return _offset_writer->estimate_buffer_size() + - (is_nullable() ? _null_writer->estimate_buffer_size() : 0) + - _item_writer->estimate_buffer_size(); + uint64_t size = _offset_writer->estimate_buffer_size() + + (is_nullable() ? _null_writer->estimate_buffer_size() : 0) + + _item_writer->estimate_buffer_size(); + if (_opts.need_inverted_index) { + DORIS_CHECK(_inverted_index_writer != nullptr); + size += _inverted_index_writer->size(); + } + if (_opts.need_ann_index) { + DORIS_CHECK(_ann_index_writer != nullptr); + size += _ann_index_writer->size(); + } + return size; } Status ArrayColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** ptr, diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index 8b7f56221cf8cf..22a7a611d37c5e 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -25,9 +25,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -790,6 +792,71 @@ TEST_F(InvertedIndexWriterTest, NumericWrite) { test_numeric_write("test_rowset_3", 0); } +TEST_F(InvertedIndexWriterTest, HighCardinalityBkdMemoryEstimate) { + auto tablet_schema = create_schema(); + + auto index_meta_pb = std::make_unique(); + index_meta_pb->set_index_type(IndexType::INVERTED); + index_meta_pb->set_index_id(1); + index_meta_pb->set_index_name("test"); + index_meta_pb->clear_col_unique_id(); + index_meta_pb->add_col_unique_id(0); + auto* properties = index_meta_pb->mutable_properties(); + (*properties)["type"] = "bkd"; + + TabletIndex idx_meta; + idx_meta.init_from_pb(*index_meta_pb.get()); + + std::string rowset_id = "test_rowset_high_cardinality_memory"; + int seg_id = 0; + std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( + local_segment_path(kTestDir, rowset_id, seg_id))}; + std::string index_path = InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix); + + io::FileWriterPtr file_writer; + io::FileWriterOptions opts; + auto fs = io::global_local_filesystem(); + Status sts = fs->create_file(index_path, &file_writer, &opts); + ASSERT_TRUE(sts.ok()) << sts; + auto index_file_writer = std::make_unique( + fs, index_path_prefix, rowset_id, seg_id, InvertedIndexStorageFormatPB::V2, + std::move(file_writer)); + + const TabletColumn& column = tablet_schema->column(0); + ASSERT_NE(&column, nullptr); + std::unique_ptr field(StorageFieldFactory::create(column)); + ASSERT_NE(field.get(), nullptr); + + std::unique_ptr column_writer; + auto status = IndexColumnWriter::create(field.get(), &column_writer, index_file_writer.get(), + &idx_meta); + ASSERT_TRUE(status.ok()) << status; + + const int64_t initial_size = column_writer->size(); + int64_t peak_size = initial_size; + constexpr size_t value_count = 4096; + constexpr size_t batch_size = 512; + std::vector values(value_count); + std::iota(values.begin(), values.end(), 0); + + for (size_t offset = 0; offset < values.size(); offset += batch_size) { + size_t count = std::min(batch_size, values.size() - offset); + status = column_writer->add_values("c1", values.data() + offset, count); + ASSERT_TRUE(status.ok()) << status; + peak_size = std::max(peak_size, column_writer->size()); + } + + EXPECT_GT(peak_size, initial_size); + EXPECT_GE(peak_size - initial_size, static_cast(values.size() * sizeof(int32_t))); + + status = column_writer->finish(); + ASSERT_TRUE(status.ok()) << status; + status = index_file_writer->begin_close(); + ASSERT_TRUE(status.ok()) << status; + status = index_file_writer->finish_close(); + ASSERT_TRUE(status.ok()) << status; +} + // Test case for Unicode string values with enable_correct_term_write=true TEST_F(InvertedIndexWriterTest, UnicodeStringWriteEnabled) { test_unicode_string_write("test_rowset_4", 0, true); @@ -1544,4 +1611,4 @@ TEST_F(InvertedIndexWriterTest, NormsFileCreationWithTokenization) { << "inverted_index_writer.cpp where .nrm file creation depends on _should_analyzer."; } -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 From 083b5325554b312da1a39303b37c31d5ec700080 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 21 May 2026 14:57:28 +0800 Subject: [PATCH 2/3] [improvement](be) Account fulltext inverted index writer memory ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: Large string full-text inverted index builds buffered CLucene postings in SDocumentsWriter, but Doris did not include that RAM in IndexColumnWriter::size(). Account the full-text writer RAM for analyzed slice indexes so segment memory estimation can see the largest buffered source, while leaving numeric/BKD and non-tokenized string paths unchanged. ### Release note None ### Check List (For Author) - Test: Unit Test - sh run-be-ut.sh --run --filter=InvertedIndexWriterTest.FullTextStringMemoryEstimateIncludesBufferedPostings - PATH=/mnt/disk6/common/ldb_toolchain_toucan/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk1/jiangkai/.bun/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk1/jiangkai/.codex/tmp/arg0/codex-arg0dYzs9D:/mnt/disk6/common/node-v24.14.1-linux-x64/lib/node_modules/@openai/codex/node_modules/@openai/codex-linux-x64/vendor/x86_64-unknown-linux-musl/path:/mnt/disk1/jiangkai/.bun/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk1/jiangkai/.bun/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/workspace/bin/lark-cli/:/mnt/disk1/jiangkai/workspace/bin/go/bin:/mnt/disk1/jiangkai/workspace/bin/ldb_toolchain/bin:/mnt/disk1/jiangkai/.local/bin:/mnt/disk1/jiangkai/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/mnt/disk6/common/apache-maven-3.9.14/bin:/mnt/disk6/common/ldb_toolchain_028/bin:/mnt/disk6/common/jdk-17.0.16/bin:/mnt/disk6/common/node-v24.14.1-linux-x64/bin:/usr/share/Modules/bin:/usr/lib64/ccache:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin build-support/check-format.sh - build-support/run-clang-tidy.sh --build-dir be/ut_build_ASAN --files be/src/storage/index/inverted/inverted_index_writer.cpp be/test/storage/segment/inverted_index_writer_test.cpp (attempted; failed on environment/pre-existing diagnostics including missing stddef.h, unmatched NOLINTEND in be/src/core/types.h, and existing full-file complexity warnings) - Behavior changed: Yes (full-text inverted index writer memory is now included in segment memory estimation; file format is unchanged) - Does this need documentation: No --- .../index/inverted/inverted_index_writer.cpp | 52 +----- .../segment/inverted_index_writer_test.cpp | 170 ++++++++++++++---- contrib/clucene | 2 +- 3 files changed, 146 insertions(+), 78 deletions(-) diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index 902c630fb84394..f90fd80bd685ec 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -37,51 +37,11 @@ const int DIMS = 1; namespace { -template -int64_t vector_memory_size(const std::vector& vector) { - return cast_set(vector.capacity() * sizeof(T)); -} - -int64_t bytes_ref_memory_size(const std::shared_ptr& bytes_ref) { - if (bytes_ref == nullptr) { - return 0; - } - return cast_set(sizeof(lucene::util::BytesRef)) + vector_memory_size(bytes_ref->bytes); -} - -int64_t heap_point_writer_memory_size( - const std::shared_ptr& writer) { - if (writer == nullptr) { - return 0; - } - - int64_t size = cast_set(sizeof(lucene::util::bkd::heap_point_writer)); - size += vector_memory_size(writer->doc_IDs_); - size += vector_memory_size(writer->ords_long_); - size += vector_memory_size(writer->ords_); - size += vector_memory_size(writer->blocks_); - for (const auto& block : writer->blocks_) { - size += vector_memory_size(block); - } - size += bytes_ref_memory_size(writer->cache_); - size += bytes_ref_memory_size(writer->cache1_); - return size; -} - -int64_t bkd_writer_memory_size(const std::shared_ptr& writer) { - if (writer == nullptr) { +int64_t index_writer_memory_size(const std::unique_ptr& index_writer) { + if (index_writer == nullptr) { return 0; } - - int64_t size = cast_set(sizeof(lucene::util::bkd::bkd_writer)); - size += vector_memory_size(writer->min_packed_value_); - size += vector_memory_size(writer->max_packed_value_); - size += vector_memory_size(writer->scratch_diff_); - size += vector_memory_size(writer->scratch1_); - size += vector_memory_size(writer->scratch2_); - size += vector_memory_size(writer->common_prefix_lengths_); - size += heap_point_writer_memory_size(writer->heap_point_writer_); - return size; + return index_writer->ramSizeInBytes(); } int64_t ram_directory_memory_size(const std::shared_ptr& dir) { @@ -635,8 +595,10 @@ Status InvertedIndexColumnWriter::add_value(const CppType& value) { template int64_t InvertedIndexColumnWriter::size() const { int64_t size = cast_set(_null_bitmap.getSizeInBytes(false)); - if constexpr (field_is_numeric_type(field_type)) { - size += bkd_writer_memory_size(_bkd_writer); + if constexpr (field_is_slice_type(field_type)) { + if (_should_analyzer) { + size += index_writer_memory_size(_index_writer); + } } size += ram_directory_memory_size(_dir); return size; diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index 22a7a611d37c5e..70e3b231e39d18 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include @@ -777,6 +777,21 @@ class InvertedIndexWriterTest : public testing::Test { std::unique_ptr _inverted_index_query_cache; }; +namespace { + +TabletIndex create_standard_fulltext_index_meta(); +std::string fulltext_memory_token(size_t token_id); +std::vector fulltext_memory_strings(size_t value_count, size_t tokens_per_value); +std::vector slices_from_strings(const std::vector& strings); +int64_t add_fulltext_values_and_get_peak_delta(IndexColumnWriter* column_writer, + const std::vector& values); +void check_fulltext_match(const std::shared_ptr& fulltext_reader, + const IndexQueryContextPtr& context, const std::string& field_name, + const std::string& query, uint64_t cardinality, + std::optional doc_id); + +} // namespace + // Test case for writing string values TEST_F(InvertedIndexWriterTest, StringWrite) { test_string_write("test_rowset_1", 0); @@ -792,22 +807,10 @@ TEST_F(InvertedIndexWriterTest, NumericWrite) { test_numeric_write("test_rowset_3", 0); } -TEST_F(InvertedIndexWriterTest, HighCardinalityBkdMemoryEstimate) { +TEST_F(InvertedIndexWriterTest, FullTextStringMemoryEstimateIncludesBufferedPostings) { auto tablet_schema = create_schema(); - - auto index_meta_pb = std::make_unique(); - index_meta_pb->set_index_type(IndexType::INVERTED); - index_meta_pb->set_index_id(1); - index_meta_pb->set_index_name("test"); - index_meta_pb->clear_col_unique_id(); - index_meta_pb->add_col_unique_id(0); - auto* properties = index_meta_pb->mutable_properties(); - (*properties)["type"] = "bkd"; - - TabletIndex idx_meta; - idx_meta.init_from_pb(*index_meta_pb.get()); - - std::string rowset_id = "test_rowset_high_cardinality_memory"; + TabletIndex idx_meta = create_standard_fulltext_index_meta(); + std::string rowset_id = "test_rowset_fulltext_memory"; int seg_id = 0; std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( local_segment_path(kTestDir, rowset_id, seg_id))}; @@ -822,7 +825,7 @@ TEST_F(InvertedIndexWriterTest, HighCardinalityBkdMemoryEstimate) { fs, index_path_prefix, rowset_id, seg_id, InvertedIndexStorageFormatPB::V2, std::move(file_writer)); - const TabletColumn& column = tablet_schema->column(0); + const TabletColumn& column = tablet_schema->column(1); ASSERT_NE(&column, nullptr); std::unique_ptr field(StorageFieldFactory::create(column)); ASSERT_NE(field.get(), nullptr); @@ -832,22 +835,16 @@ TEST_F(InvertedIndexWriterTest, HighCardinalityBkdMemoryEstimate) { &idx_meta); ASSERT_TRUE(status.ok()) << status; - const int64_t initial_size = column_writer->size(); - int64_t peak_size = initial_size; - constexpr size_t value_count = 4096; - constexpr size_t batch_size = 512; - std::vector values(value_count); - std::iota(values.begin(), values.end(), 0); - - for (size_t offset = 0; offset < values.size(); offset += batch_size) { - size_t count = std::min(batch_size, values.size() - offset); - status = column_writer->add_values("c1", values.data() + offset, count); - ASSERT_TRUE(status.ok()) << status; - peak_size = std::max(peak_size, column_writer->size()); - } + constexpr size_t value_count = 256; + constexpr size_t tokens_per_value = 48; + const auto strings = fulltext_memory_strings(value_count, tokens_per_value); + const auto values = slices_from_strings(strings); + const int64_t peak_delta = add_fulltext_values_and_get_peak_delta(column_writer.get(), values); - EXPECT_GT(peak_size, initial_size); - EXPECT_GE(peak_size - initial_size, static_cast(values.size() * sizeof(int32_t))); + RecordProperty("legacy_peak_delta_bytes", 0); + RecordProperty("peak_delta_bytes", peak_delta); + EXPECT_GT(peak_delta, 256 * 1024); + EXPECT_LT(peak_delta, 8 * 1024 * 1024); status = column_writer->finish(); ASSERT_TRUE(status.ok()) << status; @@ -855,6 +852,35 @@ TEST_F(InvertedIndexWriterTest, HighCardinalityBkdMemoryEstimate) { ASSERT_TRUE(status.ok()) << status; status = index_file_writer->finish_close(); ASSERT_TRUE(status.ok()) << status; + + auto reader = std::make_shared( + io::global_local_filesystem(), index_path_prefix, InvertedIndexStorageFormatPB::V2); + status = reader->init(); + ASSERT_EQ(status, Status::OK()); + auto result = reader->open(&idx_meta); + ASSERT_TRUE(result.has_value()) << "Failed to open compound reader" << result.error(); + + auto fulltext_reader = FullTextIndexReader::create_shared(&idx_meta, reader); + ASSERT_NE(fulltext_reader, nullptr); + + OlapReaderStatistics stats; + RuntimeState runtime_state; + TQueryOptions query_options; + query_options.enable_inverted_index_searcher_cache = false; + runtime_state.set_query_options(query_options); + io::IOContext io_ctx; + auto context = std::make_shared(); + context->io_ctx = &io_ctx; + context->stats = &stats; + context->runtime_state = &runtime_state; + std::string field_name = std::to_string(field->unique_id()); + + check_fulltext_match(fulltext_reader, context, field_name, "sharedterm", value_count, + std::nullopt); + constexpr size_t selected_row = 17; + check_fulltext_match(fulltext_reader, context, field_name, + fulltext_memory_token(selected_row * tokens_per_value + 5), 1, + selected_row); } // Test case for Unicode string values with enable_correct_term_write=true @@ -1611,4 +1637,84 @@ TEST_F(InvertedIndexWriterTest, NormsFileCreationWithTokenization) { << "inverted_index_writer.cpp where .nrm file creation depends on _should_analyzer."; } +namespace { + +TabletIndex create_standard_fulltext_index_meta() { + auto index_meta_pb = std::make_unique(); + index_meta_pb->set_index_type(IndexType::INVERTED); + index_meta_pb->set_index_id(1); + index_meta_pb->set_index_name("test"); + index_meta_pb->clear_col_unique_id(); + index_meta_pb->add_col_unique_id(1); + auto* properties = index_meta_pb->mutable_properties(); + (*properties)["parser"] = "standard"; + + TabletIndex idx_meta; + idx_meta.init_from_pb(*index_meta_pb.get()); + return idx_meta; +} + +std::string fulltext_memory_token(size_t token_id) { + std::string token = "term"; + for (int i = 0; i < 5; ++i) { + token.push_back(static_cast('a' + token_id % 26)); + token_id /= 26; + } + return token; +} + +std::vector fulltext_memory_strings(size_t value_count, size_t tokens_per_value) { + std::vector strings; + strings.reserve(value_count); + for (size_t row = 0; row < value_count; ++row) { + std::string value = "sharedterm"; + for (size_t token_idx = 0; token_idx < tokens_per_value; ++token_idx) { + value.append(" "); + value.append(fulltext_memory_token(row * tokens_per_value + token_idx)); + } + strings.emplace_back(std::move(value)); + } + return strings; +} + +std::vector slices_from_strings(const std::vector& strings) { + std::vector values; + values.reserve(strings.size()); + for (const auto& value : strings) { + values.emplace_back(value); + } + return values; +} + +int64_t add_fulltext_values_and_get_peak_delta(IndexColumnWriter* column_writer, + const std::vector& values) { + const int64_t initial_size = column_writer->size(); + int64_t peak_size = initial_size; + constexpr size_t batch_size = 32; + for (size_t offset = 0; offset < values.size(); offset += batch_size) { + size_t count = std::min(batch_size, values.size() - offset); + auto status = column_writer->add_values("c2", values.data() + offset, count); + EXPECT_TRUE(status.ok()) << status; + peak_size = std::max(peak_size, column_writer->size()); + } + return peak_size - initial_size; +} + +void check_fulltext_match(const std::shared_ptr& fulltext_reader, + const IndexQueryContextPtr& context, const std::string& field_name, + const std::string& query, uint64_t cardinality, + std::optional doc_id) { + std::shared_ptr bitmap = std::make_shared(); + StringRef query_ref(query.c_str(), query.length()); + auto status = fulltext_reader->query(context, field_name, &query_ref, + InvertedIndexQueryType::MATCH_ANY_QUERY, bitmap); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(bitmap->cardinality(), cardinality); + if (doc_id.has_value()) { + EXPECT_TRUE(bitmap->contains(doc_id.value())); + } +} + +} // namespace + } // namespace doris::segment_v2 diff --git a/contrib/clucene b/contrib/clucene index c51b5cc9adc638..bab8109b4b3e12 160000 --- a/contrib/clucene +++ b/contrib/clucene @@ -1 +1 @@ -Subproject commit c51b5cc9adc63817ad8322f617c75737ece7288d +Subproject commit bab8109b4b3e12c1b1b35247c05891051ebacc16 From 1d4861e1eaf24275678e7856534712cb06614b3c Mon Sep 17 00:00:00 2001 From: airborne12 Date: Thu, 21 May 2026 15:57:43 +0800 Subject: [PATCH 3/3] [improvement](be) Cap fulltext index memory without ram dir ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Cap CLucene buffered postings memory for analyzed inverted index writers when the actual directory is DorisFSDirectory. This reduces retained writer memory for large fulltext string values when RAM directory is disabled, while leaving RAMDirectory and untokenized index writer buffering unchanged. ### Release note Add inverted_index_ram_buffer_size_when_ram_dir_disabled to cap analyzed inverted index writer buffering when RAM directory is disabled. ### Check List (For Author) - Test: Unit Test - sh run-be-ut.sh --run --filter=InvertedIndexWriterTest.FullTextStringMemoryEstimateIncludesBufferedPostings:InvertedIndexWriterTest.FullTextLargeStringRamDirDisabledCapsBufferedPostings - PATH=/mnt/disk6/common/ldb_toolchain_toucan/bin:$PATH build-support/check-format.sh - git diff --check - CLANG_TIDY_BINARY=/mnt/disk6/common/ldb_toolchain_toucan/bin/clang-tidy-16 build-support/run-clang-tidy.sh --build-dir be/ut_build_ASAN (failed: existing clang-tidy analysis issues in jni-util.h static assertions and existing complexity warnings) - Behavior changed: Yes. Analyzed inverted index writers use the lower configurable RAM buffer cap when the actual index directory is DorisFSDirectory. - Does this need documentation: No --- be/src/common/config.cpp | 3 + be/src/common/config.h | 1 + .../index/inverted/inverted_index_writer.cpp | 18 ++- .../segment/inverted_index_writer_test.cpp | 110 +++++++++++++++++- 4 files changed, 128 insertions(+), 4 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 7b6db1636234aa..384132206c77cb 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1276,6 +1276,9 @@ DEFINE_Int32(ann_index_result_cache_stale_sweep_time_sec, "1800"); // inverted index DEFINE_mDouble(inverted_index_ram_buffer_size, "512"); +// Cap the CLucene buffered postings for analyzed inverted indexes when RAM directory is disabled. +// Values <= 0 keep inverted_index_ram_buffer_size unchanged. +DEFINE_mDouble(inverted_index_ram_buffer_size_when_ram_dir_disabled, "64"); // -1 indicates not working. // Normally we should not change this, it's useful for testing. DEFINE_mInt32(inverted_index_max_buffered_docs, "-1"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 427282a4452bc4..02354ad89747c4 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1320,6 +1320,7 @@ DECLARE_Int32(ann_index_result_cache_stale_sweep_time_sec); // inverted index DECLARE_mDouble(inverted_index_ram_buffer_size); +DECLARE_mDouble(inverted_index_ram_buffer_size_when_ram_dir_disabled); DECLARE_mInt32(inverted_index_max_buffered_docs); // dict path for chinese analyzer DECLARE_String(inverted_index_dict_path); diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index f90fd80bd685ec..9ef1d37618a028 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -17,6 +17,7 @@ #include "storage/index/inverted/inverted_index_writer.h" +#include #include #include "common/cast_set.h" @@ -58,6 +59,21 @@ int64_t ram_directory_memory_size(const std::shared_ptr& dir) return size; } +bool is_fs_directory(const std::shared_ptr& dir) { + return dir != nullptr && std::strcmp(dir->getObjectName(), "DorisFSDirectory") == 0; +} + +float index_writer_ram_buffer_size(const std::shared_ptr& dir, + bool should_analyzer) { + auto ram_buffer_size = config::inverted_index_ram_buffer_size; + if (should_analyzer && is_fs_directory(dir) && ram_buffer_size > 0 && + config::inverted_index_ram_buffer_size_when_ram_dir_disabled > 0) { + ram_buffer_size = std::min(ram_buffer_size, + config::inverted_index_ram_buffer_size_when_ram_dir_disabled); + } + return static_cast(ram_buffer_size); +} + } // namespace template @@ -169,7 +185,7 @@ InvertedIndexColumnWriter::create_index_writer() { { index_writer->setMaxBufferedDocs(1); }) DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_index_writer_setMergeFactor_error", { index_writer->setMergeFactor(1); }) - index_writer->setRAMBufferSizeMB(static_cast(config::inverted_index_ram_buffer_size)); + index_writer->setRAMBufferSizeMB(index_writer_ram_buffer_size(_dir, _should_analyzer)); index_writer->setMaxBufferedDocs(config::inverted_index_max_buffered_docs); index_writer->setMaxFieldLength(MAX_FIELD_LEN); index_writer->setMergeFactor(MERGE_FACTOR); diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index 70e3b231e39d18..7af0000d448b35 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -782,14 +782,38 @@ namespace { TabletIndex create_standard_fulltext_index_meta(); std::string fulltext_memory_token(size_t token_id); std::vector fulltext_memory_strings(size_t value_count, size_t tokens_per_value); +std::vector fulltext_large_memory_strings(size_t value_count, + size_t min_bytes_per_value); std::vector slices_from_strings(const std::vector& strings); int64_t add_fulltext_values_and_get_peak_delta(IndexColumnWriter* column_writer, - const std::vector& values); + const std::vector& values, + size_t batch_size = 32); void check_fulltext_match(const std::shared_ptr& fulltext_reader, const IndexQueryContextPtr& context, const std::string& field_name, const std::string& query, uint64_t cardinality, std::optional doc_id); +class FullTextIndexConfigGuard { +public: + FullTextIndexConfigGuard() + : _inverted_index_ram_dir_enable(config::inverted_index_ram_dir_enable), + _inverted_index_ram_buffer_size(config::inverted_index_ram_buffer_size), + _inverted_index_ram_buffer_size_when_ram_dir_disabled( + config::inverted_index_ram_buffer_size_when_ram_dir_disabled) {} + + ~FullTextIndexConfigGuard() { + config::inverted_index_ram_dir_enable = _inverted_index_ram_dir_enable; + config::inverted_index_ram_buffer_size = _inverted_index_ram_buffer_size; + config::inverted_index_ram_buffer_size_when_ram_dir_disabled = + _inverted_index_ram_buffer_size_when_ram_dir_disabled; + } + +private: + bool _inverted_index_ram_dir_enable; + double _inverted_index_ram_buffer_size; + double _inverted_index_ram_buffer_size_when_ram_dir_disabled; +}; + } // namespace // Test case for writing string values @@ -883,6 +907,70 @@ TEST_F(InvertedIndexWriterTest, FullTextStringMemoryEstimateIncludesBufferedPost selected_row); } +TEST_F(InvertedIndexWriterTest, FullTextLargeStringRamDirDisabledCapsBufferedPostings) { + FullTextIndexConfigGuard config_guard; + config::inverted_index_ram_dir_enable = false; + config::inverted_index_ram_buffer_size = 512; + + auto tablet_schema = create_schema(); + TabletIndex idx_meta = create_standard_fulltext_index_meta(); + constexpr size_t value_count = 2; + constexpr size_t min_bytes_per_value = 1024 * 1024; + const auto strings = fulltext_large_memory_strings(value_count, min_bytes_per_value); + ASSERT_GE(strings.front().size(), min_bytes_per_value); + const auto values = slices_from_strings(strings); + + auto write_and_measure = [&](std::string_view rowset_id, int seg_id, int64_t* peak_delta) { + std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( + local_segment_path(kTestDir, rowset_id, seg_id))}; + std::string index_path = InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix); + + io::FileWriterPtr file_writer; + io::FileWriterOptions opts; + auto fs = io::global_local_filesystem(); + Status sts = fs->create_file(index_path, &file_writer, &opts); + ASSERT_TRUE(sts.ok()) << sts; + auto index_file_writer = std::make_unique( + fs, index_path_prefix, std::string {rowset_id}, seg_id, + InvertedIndexStorageFormatPB::V2, std::move(file_writer)); + + const TabletColumn& column = tablet_schema->column(1); + ASSERT_NE(&column, nullptr); + std::unique_ptr field(StorageFieldFactory::create(column)); + ASSERT_NE(field.get(), nullptr); + + std::unique_ptr column_writer; + auto status = IndexColumnWriter::create(field.get(), &column_writer, + index_file_writer.get(), &idx_meta); + ASSERT_TRUE(status.ok()) << status; + + *peak_delta = add_fulltext_values_and_get_peak_delta(column_writer.get(), values, 1); + + status = column_writer->finish(); + ASSERT_TRUE(status.ok()) << status; + status = index_file_writer->begin_close(); + ASSERT_TRUE(status.ok()) << status; + status = index_file_writer->finish_close(); + ASSERT_TRUE(status.ok()) << status; + }; + + int64_t uncapped_peak_delta = 0; + config::inverted_index_ram_buffer_size_when_ram_dir_disabled = 0; + ASSERT_NO_FATAL_FAILURE( + write_and_measure("test_rowset_large_string_uncapped", 0, &uncapped_peak_delta)); + + int64_t capped_peak_delta = 0; + config::inverted_index_ram_buffer_size_when_ram_dir_disabled = 1; + ASSERT_NO_FATAL_FAILURE( + write_and_measure("test_rowset_large_string_capped", 1, &capped_peak_delta)); + + RecordProperty("uncapped_peak_delta_bytes", uncapped_peak_delta); + RecordProperty("capped_peak_delta_bytes", capped_peak_delta); + EXPECT_GT(uncapped_peak_delta, 4 * 1024 * 1024); + EXPECT_LT(capped_peak_delta, 2 * 1024 * 1024); + EXPECT_LT(capped_peak_delta * 2, uncapped_peak_delta); +} + // Test case for Unicode string values with enable_correct_term_write=true TEST_F(InvertedIndexWriterTest, UnicodeStringWriteEnabled) { test_unicode_string_write("test_rowset_4", 0, true); @@ -1677,6 +1765,22 @@ std::vector fulltext_memory_strings(size_t value_count, size_t toke return strings; } +std::vector fulltext_large_memory_strings(size_t value_count, + size_t min_bytes_per_value) { + std::vector strings; + strings.reserve(value_count); + size_t token_id = 0; + for (size_t row = 0; row < value_count; ++row) { + std::string value = "sharedterm"; + while (value.size() < min_bytes_per_value) { + value.append(" "); + value.append(fulltext_memory_token(token_id++)); + } + strings.emplace_back(std::move(value)); + } + return strings; +} + std::vector slices_from_strings(const std::vector& strings) { std::vector values; values.reserve(strings.size()); @@ -1687,10 +1791,10 @@ std::vector slices_from_strings(const std::vector& strings) } int64_t add_fulltext_values_and_get_peak_delta(IndexColumnWriter* column_writer, - const std::vector& values) { + const std::vector& values, + size_t batch_size) { const int64_t initial_size = column_writer->size(); int64_t peak_size = initial_size; - constexpr size_t batch_size = 32; for (size_t offset = 0; offset < values.size(); offset += batch_size) { size_t count = std::min(batch_size, values.size() - offset); auto status = column_writer->add_values("c2", values.data() + offset, count);