From 8b1034cd17c794385e76f46cef43cfedf2e7eec1 Mon Sep 17 00:00:00 2001 From: hoshinojyunn Date: Mon, 13 Jul 2026 19:01:15 +0800 Subject: [PATCH] [fix](be) Avoid global null bitmap in sub-reader context ### What problem does this PR solve? Issue Number: N/A Related PR: N/A Problem Summary: Multi-segment collection creates a QueryExecutionContext for each CLucene sub-reader. Copying the root NULL bitmap resolver into that context lets nullable boolean scorers interpret a table-level NULL bitmap with local docids. For example, consider two sub-readers with two documents each. In the first sub-reader, local doc 0 maps to global doc 0 and is NULL. In the second, local doc 0 maps to global doc 2. The root NULL bitmap is therefore {0}. For NOT PREFIX(title, "missing"), the second sub-reader incorrectly treats global bitmap entry 0 as its local doc 0, excludes global doc 2, and contributes only global doc 3. The collected TRUE bitmap becomes {1,3}; masking the actual root NULL bitmap {0} cannot restore doc 2, while the correct result is {1,2,3}. Keep the resolver on the root context only. The collector test covers this two-sub-reader case and verifies that the final NOT result retains global doc 2. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=MultiSegmentCollectorTest.* -j16 - Behavior changed: Yes (fixes incorrect multi-segment boolean query results with NULLs) - Does this need documentation: No --- .../query_v2/collect/multi_segment_util.h | 1 - .../query_v2/multi_segment_collector_test.cpp | 89 +++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/be/src/storage/index/inverted/query_v2/collect/multi_segment_util.h b/be/src/storage/index/inverted/query_v2/collect/multi_segment_util.h index 4e67b5aaacc6e7..870a8fb5b978e8 100644 --- a/be/src/storage/index/inverted/query_v2/collect/multi_segment_util.h +++ b/be/src/storage/index/inverted/query_v2/collect/multi_segment_util.h @@ -171,7 +171,6 @@ inline QueryExecutionContext create_segment_context(const QueryExecutionContext& } seg_ctx.binding_fields = original_ctx.binding_fields; - seg_ctx.null_resolver = original_ctx.null_resolver; return seg_ctx; } diff --git a/be/test/storage/index/inverted/query_v2/multi_segment_collector_test.cpp b/be/test/storage/index/inverted/query_v2/multi_segment_collector_test.cpp index 4a04421c78afaf..7feb12b6636d75 100644 --- a/be/test/storage/index/inverted/query_v2/multi_segment_collector_test.cpp +++ b/be/test/storage/index/inverted/query_v2/multi_segment_collector_test.cpp @@ -25,10 +25,14 @@ #include #include "io/fs/local_file_system.h" +#include "storage/index/index_iterator.h" #include "storage/index/index_query_context.h" #include "storage/index/inverted/analyzer/custom_analyzer.h" +#include "storage/index/inverted/inverted_index_cache.h" +#include "storage/index/inverted/query_v2/boolean_query/boolean_query_builder.h" #include "storage/index/inverted/query_v2/collect/doc_set_collector.h" #include "storage/index/inverted/query_v2/collect/multi_segment_util.h" +#include "storage/index/inverted/query_v2/prefix_query/prefix_query.h" #include "storage/index/inverted/query_v2/term_query/term_query.h" #include "storage/index/inverted/util/string_helper.h" @@ -112,6 +116,50 @@ static std::shared_ptr make_shared_reader( }}; } +class NullBitmapCachePolicy final : public LRUCachePolicy { +public: + explicit NullBitmapCachePolicy(std::shared_ptr bitmap) + : LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_QUERY_CACHE, 1024, + LRUCacheType::SIZE, 3600, 1, 0, true, false) { + _cache_value.bitmap = std::move(bitmap); + } + + void* value(Cache::Handle*) override { return &_cache_value; } + void release(Cache::Handle*) override {} + +private: + InvertedIndexQueryCache::CacheValue _cache_value; +}; + +class GlobalNullBitmapIterator final : public IndexIterator { +public: + explicit GlobalNullBitmapIterator(std::shared_ptr bitmap) + : _cache(std::move(bitmap)) {} + + IndexReaderPtr get_reader(IndexReaderType) const override { return nullptr; } + Status read_from_index(const IndexParam&) override { return Status::OK(); } + Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) override { + *cache_handle = InvertedIndexQueryCacheHandle(&_cache, reinterpret_cast(1)); + return Status::OK(); + } + Result has_null() override { return true; } + +private: + NullBitmapCachePolicy _cache; +}; + +class GlobalNullBitmapResolver final : public NullBitmapResolver { +public: + explicit GlobalNullBitmapResolver(GlobalNullBitmapIterator* iterator) : _iterator(iterator) {} + + IndexIterator* iterator_for(const Scorer&, const std::string&) const override { + return _iterator; + } + +private: + GlobalNullBitmapIterator* _iterator; +}; + TEST_F(MultiSegmentCollectorTest, CollectDocSetWithMultiReader) { auto* dir0 = FSDirectory::getDirectory((kTestDir + "/segment0").c_str()); auto* dir1 = FSDirectory::getDirectory((kTestDir + "/segment1").c_str()); @@ -213,4 +261,45 @@ TEST_F(MultiSegmentCollectorTest, CollectDocSetWithSingleReaderBinding) { _CLDECDELETE(dir1); } +TEST_F(MultiSegmentCollectorTest, CollectDocSetDoesNotUseGlobalNullBitmapInSubReader) { + auto* dir0 = FSDirectory::getDirectory((kTestDir + "/segment0").c_str()); + auto* dir1 = FSDirectory::getDirectory((kTestDir + "/segment1").c_str()); + + ValueArray readers(2); + readers[0] = lucene::index::IndexReader::open(dir0, true); + readers[1] = lucene::index::IndexReader::open(dir1, true); + auto reader = make_shared_reader(_CLNEW lucene::index::MultiReader(&readers, true)); + + auto index_query_context = std::make_shared(); + auto field = StringHelper::to_wstring("title"); + auto prefix_query = std::make_shared(index_query_context, field, "missing"); + OperatorBooleanQueryBuilder query_builder(OperatorType::OP_NOT); + query_builder.add(prefix_query, "title-binding"); + auto weight = query_builder.build()->weight(false); + + auto global_null_bitmap = std::make_shared(); + global_null_bitmap->add(0); + GlobalNullBitmapIterator null_iterator(global_null_bitmap); + GlobalNullBitmapResolver null_resolver(&null_iterator); + + QueryExecutionContext exec_ctx; + exec_ctx.segment_num_rows = reader->maxDoc(); + exec_ctx.readers = {reader}; + exec_ctx.reader_bindings.emplace("title-binding", reader); + exec_ctx.field_reader_bindings.emplace(field, reader); + exec_ctx.null_resolver = &null_resolver; + + auto roaring = std::make_shared(); + collect_multi_segment_doc_set(weight, exec_ctx, "title-binding", roaring, nullptr, false); + *roaring -= *global_null_bitmap; + + EXPECT_EQ(roaring->cardinality(), 3); + EXPECT_TRUE(roaring->contains(1)); + EXPECT_TRUE(roaring->contains(2)); + EXPECT_TRUE(roaring->contains(3)); + + _CLDECDELETE(dir0); + _CLDECDELETE(dir1); +} + } // namespace doris::segment_v2