Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,6 @@ inline QueryExecutionContext create_segment_context(const QueryExecutionContext&
}

seg_ctx.binding_fields = original_ctx.binding_fields;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This fixes the full doc-set case, but the same segment context is used by collect_multi_segment_top_k(). In that path we build scorers from seg_ctx, keep only k docs per segment and then globally, and only later does FunctionSearch rebuild a global scorer and call mask_out_null(). Because this context no longer has a resolver, nullable NOT/boolean scorers treat NULL rows as ordinary matches during top-k selection; after the later mask removes those rows, the next non-NULL candidates have already been discarded. A concrete shape is top_k=3 over docs 0..3 where doc 0 is NULL for the field and the query is NOT missing: top-k can retain 0,1,2 on equal scores, the global mask drops 0, and doc 3 is never reconsidered. Please either filter/mark NULL rows before top-k truncation using segment-local null bitmaps, or apply the original global NULL mask before enforcing the final k, and add a scored descending top-k regression.

seg_ctx.null_resolver = original_ctx.null_resolver;

return seg_ctx;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,14 @@
#include <vector>

#include "io/fs/local_file_system.h"
#include "storage/index/index_iterator.h"
#include "storage/index/index_query_context.h"
#include "storage/index/inverted/analyzer/custom_analyzer.h"
#include "storage/index/inverted/inverted_index_cache.h"
#include "storage/index/inverted/query_v2/boolean_query/boolean_query_builder.h"
#include "storage/index/inverted/query_v2/collect/doc_set_collector.h"
#include "storage/index/inverted/query_v2/collect/multi_segment_util.h"
#include "storage/index/inverted/query_v2/prefix_query/prefix_query.h"
#include "storage/index/inverted/query_v2/term_query/term_query.h"
#include "storage/index/inverted/util/string_helper.h"

Expand Down Expand Up @@ -112,6 +116,50 @@ static std::shared_ptr<lucene::index::IndexReader> make_shared_reader(
}};
}

class NullBitmapCachePolicy final : public LRUCachePolicy {
public:
explicit NullBitmapCachePolicy(std::shared_ptr<roaring::Roaring> bitmap)
: LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_QUERY_CACHE, 1024,
LRUCacheType::SIZE, 3600, 1, 0, true, false) {
_cache_value.bitmap = std::move(bitmap);
}

void* value(Cache::Handle*) override { return &_cache_value; }
void release(Cache::Handle*) override {}

private:
InvertedIndexQueryCache::CacheValue _cache_value;
};

class GlobalNullBitmapIterator final : public IndexIterator {
public:
explicit GlobalNullBitmapIterator(std::shared_ptr<roaring::Roaring> bitmap)
: _cache(std::move(bitmap)) {}

IndexReaderPtr get_reader(IndexReaderType) const override { return nullptr; }
Status read_from_index(const IndexParam&) override { return Status::OK(); }
Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) override {
*cache_handle = InvertedIndexQueryCacheHandle(&_cache, reinterpret_cast<Cache::Handle*>(1));
return Status::OK();
}
Result<bool> has_null() override { return true; }

private:
NullBitmapCachePolicy _cache;
};

class GlobalNullBitmapResolver final : public NullBitmapResolver {
public:
explicit GlobalNullBitmapResolver(GlobalNullBitmapIterator* iterator) : _iterator(iterator) {}

IndexIterator* iterator_for(const Scorer&, const std::string&) const override {
return _iterator;
}

private:
GlobalNullBitmapIterator* _iterator;
};

TEST_F(MultiSegmentCollectorTest, CollectDocSetWithMultiReader) {
auto* dir0 = FSDirectory::getDirectory((kTestDir + "/segment0").c_str());
auto* dir1 = FSDirectory::getDirectory((kTestDir + "/segment1").c_str());
Expand Down Expand Up @@ -213,4 +261,45 @@ TEST_F(MultiSegmentCollectorTest, CollectDocSetWithSingleReaderBinding) {
_CLDECDELETE(dir1);
}

TEST_F(MultiSegmentCollectorTest, CollectDocSetDoesNotUseGlobalNullBitmapInSubReader) {
auto* dir0 = FSDirectory::getDirectory((kTestDir + "/segment0").c_str());
auto* dir1 = FSDirectory::getDirectory((kTestDir + "/segment1").c_str());

ValueArray<lucene::index::IndexReader*> readers(2);
readers[0] = lucene::index::IndexReader::open(dir0, true);
readers[1] = lucene::index::IndexReader::open(dir1, true);
auto reader = make_shared_reader(_CLNEW lucene::index::MultiReader(&readers, true));

auto index_query_context = std::make_shared<IndexQueryContext>();
auto field = StringHelper::to_wstring("title");
auto prefix_query = std::make_shared<PrefixQuery>(index_query_context, field, "missing");
OperatorBooleanQueryBuilder query_builder(OperatorType::OP_NOT);
query_builder.add(prefix_query, "title-binding");
auto weight = query_builder.build()->weight(false);

auto global_null_bitmap = std::make_shared<roaring::Roaring>();
global_null_bitmap->add(0);
GlobalNullBitmapIterator null_iterator(global_null_bitmap);
GlobalNullBitmapResolver null_resolver(&null_iterator);

QueryExecutionContext exec_ctx;
exec_ctx.segment_num_rows = reader->maxDoc();
exec_ctx.readers = {reader};
exec_ctx.reader_bindings.emplace("title-binding", reader);
exec_ctx.field_reader_bindings.emplace(field, reader);
exec_ctx.null_resolver = &null_resolver;

auto roaring = std::make_shared<roaring::Roaring>();
collect_multi_segment_doc_set(weight, exec_ctx, "title-binding", roaring, nullptr, false);
*roaring -= *global_null_bitmap;

EXPECT_EQ(roaring->cardinality(), 3);
EXPECT_TRUE(roaring->contains(1));
EXPECT_TRUE(roaring->contains(2));
EXPECT_TRUE(roaring->contains(3));

_CLDECDELETE(dir0);
_CLDECDELETE(dir1);
}

} // namespace doris::segment_v2
Loading