diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index f0455343b..38ef0fc64 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -42,3 +42,4 @@ User Guide user_guide/prefetch user_guide/arrow user_guide/global_index + user_guide/primary_key_global_index diff --git a/docs/source/user_guide/primary_key_global_index.rst b/docs/source/user_guide/primary_key_global_index.rst new file mode 100644 index 000000000..3b9aaa56e --- /dev/null +++ b/docs/source/user_guide/primary_key_global_index.rst @@ -0,0 +1,73 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Primary Key Global Index +======================== + +Paimon 2.0 primary-key tables support *source-backed* global scalar indexes +(``pk-btree`` / ``pk-bitmap``). Unlike the Data Evolution global indexes described in +:doc:`global_index`, which address rows by a table-wide row id, a source-backed payload +covers the complete active source set of one positive data level of one bucket, and its +results are group ordinals that are localized back to per-file physical row positions. + +paimon-cpp supports the read path of this protocol: ordinary batch scans of a +primary-key table with scalar index definitions automatically evaluate the part of the +scan predicate that touches indexed fields against the validated payload groups of the +scanned snapshot, and narrow covered files to indexed splits carrying file-local row +ranges. No dedicated query API is required. + +Table requirements +------------------ + +The definitions follow the Java table options: + +- ``'pk-btree.index.columns' = 'price'`` with optional + ``'fields.price.pk-btree.index.options' = '{"block-size":"64 kb"}'`` +- fixed bucket (``bucket > 0``) or postpone bucket mode +- ``'deletion-vectors.enabled' = 'true'`` and ``'deletion-vectors.merge-on-read' = 'false'`` + +Semantics +--------- + +- A payload is only used when it provably covers the current active source set of its + data level: exactly one payload per level, source file names / order / row counts + identical to the active COMPACT files of that level, matching index type and field id, + and a row range of exactly ``[0, total source rows - 1]``. Anything else is treated as + uncovered and scanned normally. +- ``AND`` predicates narrow with any safely evaluable indexed child; ``OR`` predicates + only use the index when every branch is evaluable. Files whose evaluation fails, whose + positions are out of range, or whose result needs more than 4096 ranges fall back to a + normal scan individually. +- Indexed splits keep their deletion files aligned with the data file; the reader still + applies deletion vectors and the complete original predicate, so index results never + change visibility semantics. +- ``'global-index.enabled' = 'false'`` disables the planner. + +Current scope +------------- + +- The BTree payload reader is wired up. ``pk-bitmap`` (and vector / full-text) + definitions are recognized for validation, but their evaluation conservatively falls + back to a normal scan until their dedicated payload readers are supported. +- The read path targets the Java release-2.0.0 layout and scan semantics (source metadata + v1, ``GlobalIndexMeta`` with ``_SOURCE_META``, commit message v12). Source-file names + currently use the existing C++ length-prefixed UTF-8 streams; ASCII and non-null BMP + names are compatible with Java ``writeUTF``, while complete modified UTF-8 support for + supplementary code points will be handled by a shared stream-level change. +- ``PkSortedIndexFile::Build`` can build one payload for an ordered source group from + value-sorted input, which supports tooling and tests; automatic build and maintenance + during compaction is not included yet. diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 8ec150e94..9fcf8e34d 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -522,6 +522,18 @@ struct PAIMON_EXPORT Options { /// "global-index.external-path" - Global index root directory, if not set, the global index /// files will be stored under the index directory. static const char GLOBAL_INDEX_EXTERNAL_PATH[]; + /// "pk-btree.index.columns" - Comma-separated columns indexed by primary-key BTree indexes. + /// No default value. + static const char PK_BTREE_INDEX_COLUMNS[]; + /// "pk-bitmap.index.columns" - Comma-separated columns indexed by primary-key Bitmap indexes. + /// No default value. + static const char PK_BITMAP_INDEX_COLUMNS[]; + /// "pk-vector.index.columns" - Comma-separated VECTOR columns indexed by primary-key vector + /// indexes. No default value. + static const char PK_VECTOR_INDEX_COLUMNS[]; + /// "pk-full-text.index.columns" - Comma-separated character columns indexed by primary-key + /// full-text indexes. No default value. + static const char PK_FULL_TEXT_INDEX_COLUMNS[]; /// "aggregation.remove-record-on-delete" - Whether to remove the whole row in aggregation /// engine when delete records are received. Default value is "false". static const char AGGREGATION_REMOVE_RECORD_ON_DELETE[]; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 2f2537d98..b0fe91b08 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -254,6 +254,11 @@ set(PAIMON_CORE_SRCS core/index/index_file_handler.cpp core/index/global_index_meta.cpp core/index/index_file_meta_serializer.cpp + core/index/pk/primary_key_index_source_meta.cpp + core/index/pk/primary_key_index_definitions.cpp + core/index/pksorted/pk_sorted_index_group.cpp + core/index/pksorted/pk_sorted_bucket_index_state.cpp + core/index/pksorted/pk_sorted_index_file.cpp core/io/generic_row_to_arrow_array_converter.cpp core/io/meta_to_arrow_array_converter.cpp core/io/async_key_value_producer_and_consumer.cpp @@ -405,6 +410,9 @@ set(PAIMON_CORE_SRCS core/table/source/table_read.cpp core/table/source/table_scan.cpp core/table/source/data_evolution_batch_scan.cpp + core/table/source/primary_key_sorted_index_scan.cpp + core/table/source/primary_key_sorted_index_result.cpp + core/table/source/primary_key_index_batch_scan.cpp core/table/system/audit_log_system_table.cpp core/table/system/binlog_system_table.cpp core/table/system/global_system_tables.cpp @@ -723,6 +731,9 @@ if(PAIMON_BUILD_TESTS) core/index/index_in_data_file_dir_path_factory_test.cpp core/index/deletion_vector_meta_test.cpp core/index/index_file_meta_serializer_test.cpp + core/index/pk/primary_key_index_source_meta_test.cpp + core/index/pk/primary_key_index_definitions_test.cpp + core/index/pksorted/pk_sorted_bucket_index_state_test.cpp core/index/index_file_handler_test.cpp core/io/compact_increment_test.cpp core/io/infer_shredding_file_writer_test.cpp @@ -740,6 +751,7 @@ if(PAIMON_BUILD_TESTS) core/io/file_index_evaluator_test.cpp core/io/single_file_writer_test.cpp core/io/rolling_blob_file_writer_test.cpp + core/global_index/global_index_evaluator_impl_test.cpp core/global_index/indexed_split_test.cpp core/manifest/file_source_test.cpp core/manifest/file_kind_test.cpp @@ -859,6 +871,7 @@ if(PAIMON_BUILD_TESTS) core/table/sink/commit_message_test.cpp core/table/sink/commit_message_impl_test.cpp core/table/source/fallback_data_split_test.cpp + core/table/source/primary_key_sorted_index_scan_test.cpp core/table/source/table_read_test.cpp core/table/source/append_count_reader_test.cpp core/table/source/pk_count_reader_test.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 5c8674b7c..bac4f16f7 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -139,6 +139,10 @@ const char Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE[] = "blob-write-null-on-fet const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled"; const char Options::GLOBAL_INDEX_THREAD_NUM[] = "global-index.thread-num"; const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path"; +const char Options::PK_BTREE_INDEX_COLUMNS[] = "pk-btree.index.columns"; +const char Options::PK_BITMAP_INDEX_COLUMNS[] = "pk-bitmap.index.columns"; +const char Options::PK_VECTOR_INDEX_COLUMNS[] = "pk-vector.index.columns"; +const char Options::PK_FULL_TEXT_INDEX_COLUMNS[] = "pk-full-text.index.columns"; const char Options::AGGREGATION_REMOVE_RECORD_ON_DELETE[] = "aggregation.remove-record-on-delete"; const char Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED[] = "table-read.sequence-number.enabled"; const char Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED[] = "key-value.sequence_number.enabled"; diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp index c76b3ea89..84b451758 100644 --- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp +++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp @@ -783,7 +783,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -808,7 +809,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -833,7 +835,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -848,7 +851,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->FirstKey()); @@ -870,7 +874,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp index cedfa4a73..aa2f23c71 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp @@ -19,22 +19,70 @@ #include "paimon/common/global_index/btree/btree_file_meta_selector.h" +#include "fmt/format.h" #include "paimon/common/memory/memory_slice.h" namespace paimon { -BTreeFileMetaSelector::BTreeFileMetaSelector(const std::vector& files, - const std::shared_ptr& key_type, - const std::shared_ptr& pool) - : key_type_(key_type), - pool_(pool), - comparator_(KeySerializer::CreateComparator(key_type, pool)) { - files_.reserve(files.size()); +Result> BTreeFileMetaSelector::Create( + const std::vector& files, const std::shared_ptr& key_type, + const std::shared_ptr& pool) { + if (key_type == nullptr) { + return Status::Invalid("Cannot create a BTree file metadata selector without a key type."); + } + if (pool == nullptr) { + return Status::Invalid( + "Cannot create a BTree file metadata selector without a memory pool."); + } + std::vector>> decoded_files; + decoded_files.reserve(files.size()); + MemorySlice::SliceComparator comparator = KeySerializer::CreateComparator(key_type, pool); for (const auto& file : files) { - auto index_meta = BTreeIndexMeta::Deserialize(file.metadata, pool.get()); - files_.emplace_back(file, std::move(index_meta)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_meta, + BTreeIndexMeta::Deserialize(file.metadata, pool.get())); + bool has_first_key = index_meta->FirstKey() != nullptr; + bool has_last_key = index_meta->LastKey() != nullptr; + if (has_first_key != has_last_key) { + return Status::Invalid(fmt::format( + "BTree index metadata for {} must contain both boundary keys or neither.", + file.file_path)); + } + if (!has_first_key && !index_meta->HasNulls()) { + return Status::Invalid( + fmt::format("BTree index metadata for {} has no boundary keys or null values.", + file.file_path)); + } + if (index_meta->FirstKey() != nullptr) { + PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey( + WrapKeySlice(index_meta->FirstKey()), key_type)); + } + if (index_meta->LastKey() != nullptr) { + PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey( + WrapKeySlice(index_meta->LastKey()), key_type)); + } + if (index_meta->FirstKey() != nullptr && index_meta->LastKey() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(int32_t comparison, + comparator(WrapKeySlice(index_meta->FirstKey()), + WrapKeySlice(index_meta->LastKey()))); + if (comparison > 0) { + return Status::Invalid(fmt::format( + "BTree index metadata for {} has a first key greater than its last key.", + file.file_path)); + } + } + decoded_files.emplace_back(file, std::move(index_meta)); } + return std::unique_ptr( + new BTreeFileMetaSelector(std::move(decoded_files), key_type, pool)); } +BTreeFileMetaSelector::BTreeFileMetaSelector( + std::vector>> files, + std::shared_ptr key_type, std::shared_ptr pool) + : files_(std::move(files)), + key_type_(std::move(key_type)), + pool_(std::move(pool)), + comparator_(KeySerializer::CreateComparator(key_type_, pool_)) {} + Result> BTreeFileMetaSelector::VisitIsNotNull() { return Filter([](const BTreeIndexMeta& meta) -> Result { return !meta.OnlyNulls(); }); } @@ -208,7 +256,9 @@ MemorySlice BTreeFileMetaSelector::WrapKeySlice(const std::shared_ptr& ke Result BTreeFileMetaSelector::SerializeLiteral(const Literal& literal) const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, KeySerializer::SerializeKey(literal, key_type_, pool_.get())); - return MemorySlice::Wrap(bytes); + MemorySlice slice = MemorySlice::Wrap(bytes); + PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(slice, key_type_)); + return slice; } } // namespace paimon diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.h b/src/paimon/common/global_index/btree/btree_file_meta_selector.h index b59fc7ed7..ad066278e 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.h +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector.h @@ -34,9 +34,9 @@ namespace paimon { /// Selects candidate BTree index files based on filter predicates. class BTreeFileMetaSelector : public FunctionVisitor> { public: - BTreeFileMetaSelector(const std::vector& files, - const std::shared_ptr& key_type, - const std::shared_ptr& pool); + static Result> Create( + const std::vector& files, + const std::shared_ptr& key_type, const std::shared_ptr& pool); Result> VisitIsNotNull() override; Result> VisitIsNull() override; @@ -55,6 +55,10 @@ class BTreeFileMetaSelector : public FunctionVisitor> VisitLike(const Literal& literal) override; private: + BTreeFileMetaSelector( + std::vector>> files, + std::shared_ptr key_type, std::shared_ptr pool); + using MetaPredicate = std::function(const BTreeIndexMeta&)>; Result> Filter(const MetaPredicate& predicate) const; diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp index ce77207bd..bb025c118 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp @@ -87,99 +87,111 @@ class BTreeFileMetaSelectorTest : public ::testing::Test { }; TEST_F(BTreeFileMetaSelectorTest, TestVisitLessThan) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // minKey < 8: file1(1), file4(1) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitLessThan(Literal(8))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitLessThan(Literal(8))); CheckResult(result, {"file1", "file4"}); // minKey < 15: file1(1), file4(1) (file2 minKey=15, not < 15) - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(15))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessThan(Literal(15))); CheckResult(result, {"file1", "file4"}); // minKey < 1: no file has minKey < 1 - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(1))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessThan(Literal(1))); ASSERT_TRUE(result.empty()); } TEST_F(BTreeFileMetaSelectorTest, TestVisitLessOrEqual) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // minKey <= 20: file1(1), file2(15), file4(1), file5(19) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitLessOrEqual(Literal(20))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitLessOrEqual(Literal(20))); CheckResult(result, {"file1", "file2", "file4", "file5"}); // minKey <= 15: file1(1), file2(15), file4(1) - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessOrEqual(Literal(15))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessOrEqual(Literal(15))); CheckResult(result, {"file1", "file2", "file4"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterThan) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // maxKey > 20: file3(30), file5(25) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitGreaterThan(Literal(20))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitGreaterThan(Literal(20))); CheckResult(result, {"file3", "file5"}); // maxKey > 30: no file - ASSERT_OK_AND_ASSIGN(result, selector.VisitGreaterThan(Literal(30))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitGreaterThan(Literal(30))); ASSERT_TRUE(result.empty()); } TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterOrEqual) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // maxKey >= 5: all non-null files (file1..file5) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitGreaterOrEqual(Literal(5))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitGreaterOrEqual(Literal(5))); CheckResult(result, {"file1", "file2", "file3", "file4", "file5"}); // maxKey >= 20: file2(20), file3(30), file5(25) - ASSERT_OK_AND_ASSIGN(result, selector.VisitGreaterOrEqual(Literal(20))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitGreaterOrEqual(Literal(20))); CheckResult(result, {"file2", "file3", "file5"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitEqual) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // 22 in [21,30] and [19,25] - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitEqual(Literal(22))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitEqual(Literal(22))); CheckResult(result, {"file3", "file5"}); // 30 in [21,30] only - ASSERT_OK_AND_ASSIGN(result, selector.VisitEqual(Literal(30))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitEqual(Literal(30))); CheckResult(result, {"file3"}); // 100 out of all ranges - ASSERT_OK_AND_ASSIGN(result, selector.VisitEqual(Literal(100))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitEqual(Literal(100))); ASSERT_TRUE(result.empty()); + + // A mismatched literal must fail before the fixed-width comparator reads it. + ASSERT_NOK(selector->VisitEqual(Literal(static_cast(1)))); } TEST_F(BTreeFileMetaSelectorTest, TestVisitNotEqual) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // NotEqual cannot prune any file, returns all - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitNotEqual(Literal(22))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitNotEqual(Literal(22))); CheckResult(result, {"file1", "file2", "file3", "file4", "file5", "file6"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitIsNull) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // has_nulls: file1, file3, file5, file6 - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitIsNull()); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIsNull()); CheckResult(result, {"file1", "file3", "file5", "file6"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitIsNotNull) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // !onlyNulls: file1..file5 (file6 is only-nulls) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitIsNotNull()); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIsNotNull()); CheckResult(result, {"file1", "file2", "file3", "file4", "file5"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitIn) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // IN(1, 2, 3, 26, 27, 28): // 1 in [1,10]=file1, [1,5]=file4 @@ -188,42 +200,44 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitIn) { // 26 in [21,30]=file3 // 27 in [21,30]=file3 // 28 in [21,30]=file3 - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitIn({Literal(1), Literal(2), Literal(3), - Literal(26), Literal(27), Literal(28)})); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIn({Literal(1), Literal(2), Literal(3), + Literal(26), Literal(27), Literal(28)})); CheckResult(result, {"file1", "file3", "file4"}); // IN(100): no match - ASSERT_OK_AND_ASSIGN(result, selector.VisitIn({Literal(100)})); + ASSERT_OK_AND_ASSIGN(result, selector->VisitIn({Literal(100)})); ASSERT_TRUE(result.empty()); } TEST_F(BTreeFileMetaSelectorTest, TestVisitNotIn) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // NotIn cannot prune any file ASSERT_OK_AND_ASSIGN(auto result, - selector.VisitNotIn({Literal(1), Literal(7), Literal(19), Literal(30)})); + selector->VisitNotIn({Literal(1), Literal(7), Literal(19), Literal(30)})); CheckResult(result, {"file1", "file2", "file3", "file4", "file5", "file6"}); } TEST_F(BTreeFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // file6 is only-nulls, should be excluded from all range/equality queries - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitEqual(Literal(1))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitEqual(Literal(1))); auto names = FileNames(result); ASSERT_EQ(names.count("file6"), 0u); - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(100))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessThan(Literal(100))); names = FileNames(result); ASSERT_EQ(names.count("file6"), 0u); - ASSERT_OK_AND_ASSIGN(result, selector.VisitGreaterThan(Literal(0))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitGreaterThan(Literal(0))); names = FileNames(result); ASSERT_EQ(names.count("file6"), 0u); // But IsNull should include file6 - ASSERT_OK_AND_ASSIGN(result, selector.VisitIsNull()); + ASSERT_OK_AND_ASSIGN(result, selector->VisitIsNull()); names = FileNames(result); ASSERT_EQ(names.count("file6"), 1u); } @@ -249,22 +263,57 @@ TEST_F(BTreeFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) { GlobalIndexIOMeta("file_nulls", 1, null_meta->Serialize(pool.get())), }; - BTreeFileMetaSelector selector(files, key_type, pool); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files, key_type, pool)); ASSERT_OK_AND_ASSIGN(std::vector result, - selector.VisitEqual(Literal(FieldType::STRING, "www.example.com", 15))); + selector->VisitEqual(Literal(FieldType::STRING, "www.example.com", 15))); CheckResult(result, {"file_empty", "file_normal"}); - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(FieldType::STRING, "bbb.com", 7))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessThan(Literal(FieldType::STRING, "bbb.com", 7))); CheckResult(result, {"file_empty", "file_normal"}); ASSERT_OK_AND_ASSIGN( - result, selector.VisitGreaterThan(Literal(FieldType::STRING, "www.example.com", 15))); + result, selector->VisitGreaterThan(Literal(FieldType::STRING, "www.example.com", 15))); CheckResult(result, {"file_normal"}); - ASSERT_OK_AND_ASSIGN(result, selector.VisitIn({Literal(FieldType::STRING, "", 0), - Literal(FieldType::STRING, "zzz.com", 7)})); + ASSERT_OK_AND_ASSIGN(result, selector->VisitIn({Literal(FieldType::STRING, "", 0), + Literal(FieldType::STRING, "zzz.com", 7)})); CheckResult(result, {"file_empty", "file_normal"}); } +TEST_F(BTreeFileMetaSelectorTest, RejectsMalformedFileMetadata) { + std::shared_ptr pool = GetDefaultPool(); + std::vector files = {GlobalIndexIOMeta("missing", 1, /*metadata=*/nullptr)}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto truncated = std::make_shared(std::string(4, '\0'), pool.get()); + files = {GlobalIndexIOMeta("truncated", 1, truncated)}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto short_key = std::make_shared(std::string(1, '\0'), pool.get()); + auto invalid_key_meta = std::make_shared(short_key, short_key, false); + files = {GlobalIndexIOMeta("invalid-key", 1, invalid_key_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto reversed_meta = std::make_shared(SerializeInt(10), SerializeInt(1), false); + files = {GlobalIndexIOMeta("reversed-range", 1, reversed_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto only_first_meta = + std::make_shared(SerializeInt(1), /*last_key=*/nullptr, false); + files = {GlobalIndexIOMeta("only-first-key", 1, only_first_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto only_last_meta = + std::make_shared(/*first_key=*/nullptr, SerializeInt(1), false); + files = {GlobalIndexIOMeta("only-last-key", 1, only_last_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto empty_nonnull_meta = + std::make_shared(/*first_key=*/nullptr, /*last_key=*/nullptr, false); + files = {GlobalIndexIOMeta("empty-nonnull", 1, empty_nonnull_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); +} + } // namespace paimon::test diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index 29ea456d1..653b617d3 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -16,6 +16,9 @@ * specific language governing permissions and limitations * under the License. */ +#include +#include + #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" @@ -487,8 +490,8 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteEmptyStringKeyMetadata) { ASSERT_OK_AND_ASSIGN(auto metas, writer->Finish()); ASSERT_EQ(metas.size(), 1); - std::shared_ptr meta = - BTreeIndexMeta::Deserialize(metas[0].metadata, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(metas[0].metadata, pool_.get())); ASSERT_TRUE(meta->FirstKey()); ASSERT_EQ(meta->FirstKey()->size(), 0); ASSERT_TRUE(meta->LastKey()); diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp index 2ed879875..f9a4868ff 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp @@ -291,6 +291,8 @@ Result BTreeGlobalIndexReader::RangeQuery(const std::optional= from, so skip lower bound comparison. diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index e554985db..8996fe59d 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -18,10 +18,12 @@ */ #include "paimon/common/global_index/btree/btree_global_indexer.h" +#include #include #include #include "arrow/c/bridge.h" +#include "fmt/format.h" #include "paimon/common/compression/block_compression_factory.h" #include "paimon/common/global_index/btree/btree_file_footer.h" #include "paimon/common/global_index/btree/btree_global_index_writer.h" @@ -41,6 +43,7 @@ #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/memory/bytes.h" #include "paimon/utils/roaring_bitmap64.h" + namespace paimon { Result> BTreeGlobalIndexer::Create( const std::map& options) { @@ -123,10 +126,17 @@ Result> BTreeGlobalIndexer::CreateReader( } read_buffer_size = static_cast(tmp_buffer_size); } - // TODO(lisizhuo.lsz): Allow users to specify an executor - std::shared_ptr executor = CreateDefaultExecutor(); - return std::make_shared(read_buffer_size, files, key_type, file_reader, - cache_manager_, pool, executor); + // UnionGlobalIndexReader evaluates one payload inline. Preserve the existing private pool only + // when multiple payloads can submit work. + std::shared_ptr executor; + if (files.size() > 1) { + executor = CreateDefaultExecutor(); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr reader, + LazyFilteredBTreeReader::Create(read_buffer_size, files, key_type, file_reader, + cache_manager_, pool, executor)); + return std::shared_ptr(std::move(reader)); } } // namespace paimon diff --git a/src/paimon/common/global_index/btree/btree_index_meta.cpp b/src/paimon/common/global_index/btree/btree_index_meta.cpp index 9ba88e534..4b6f6ce0e 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.cpp +++ b/src/paimon/common/global_index/btree/btree_index_meta.cpp @@ -19,12 +19,28 @@ #include "paimon/common/global_index/btree/btree_index_meta.h" +#include +#include + +#include "fmt/format.h" #include "paimon/common/memory/memory_slice_output.h" namespace paimon { namespace { -std::shared_ptr ReadKey(MemorySliceInput* input, int32_t key_length, MemoryPool* pool) { +Result> ReadKey(MemorySliceInput* input, int32_t key_length, + int32_t required_remaining, const char* key_name, + MemoryPool* pool) { + if (key_length < 0) { + return Status::Invalid( + fmt::format("BTree index metadata has a negative {} length {}.", key_name, key_length)); + } + if (input->Available() < required_remaining || + key_length > input->Available() - required_remaining) { + return Status::Invalid( + fmt::format("BTree index metadata {} length {} exceeds the available payload bytes.", + key_name, key_length)); + } if (key_length == 0) { return std::make_shared(0, pool); } @@ -33,32 +49,77 @@ std::shared_ptr ReadKey(MemorySliceInput* input, int32_t key_length, Memo } // namespace -std::shared_ptr BTreeIndexMeta::Deserialize(const std::shared_ptr& meta, - paimon::MemoryPool* pool) { +Result> BTreeIndexMeta::Deserialize( + const std::shared_ptr& meta, paimon::MemoryPool* pool) { + if (meta == nullptr) { + return Status::Invalid("Cannot deserialize BTree index metadata from a null buffer."); + } + if (pool == nullptr) { + return Status::Invalid("Cannot deserialize BTree index metadata with a null memory pool."); + } + // Legacy metadata contains two int32 lengths and one has-nulls byte. + constexpr size_t kMinimumMetadataSize = 2 * sizeof(int32_t) + sizeof(int8_t); + if (meta->size() < kMinimumMetadataSize) { + return Status::Invalid(fmt::format( + "BTree index metadata is truncated: expected at least {} bytes, but found {}.", + kMinimumMetadataSize, meta->size())); + } + if (meta->size() > static_cast(std::numeric_limits::max())) { + return Status::Invalid( + fmt::format("BTree index metadata size {} exceeds the supported maximum {}.", + meta->size(), std::numeric_limits::max())); + } + MemorySlice slice = MemorySlice::Wrap(meta); MemorySliceInput input = slice.ToInput(); int32_t first_key_len = input.ReadInt(); - std::shared_ptr first_key = ReadKey(&input, first_key_len, pool); + constexpr int32_t kRequiredAfterFirstKey = sizeof(int32_t) + sizeof(int8_t); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr first_key, + ReadKey(&input, first_key_len, kRequiredAfterFirstKey, "first key", pool)); int32_t last_key_len = input.ReadInt(); - std::shared_ptr last_key = ReadKey(&input, last_key_len, pool); - bool has_nulls = input.ReadByte() == static_cast(1); + constexpr int32_t kRequiredAfterLastKey = sizeof(int8_t); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr last_key, + ReadKey(&input, last_key_len, kRequiredAfterLastKey, "last key", pool)); + int8_t has_nulls_byte = input.ReadByte(); + if (has_nulls_byte != 0 && has_nulls_byte != 1) { + return Status::Invalid( + fmt::format("BTree index metadata has invalid has-nulls value {}.", has_nulls_byte)); + } + bool has_nulls = has_nulls_byte == 1; - if (input.Available() >= 2) { + if (input.Available() == 2) { int8_t format_version = input.ReadByte(); - if (format_version == kFormatVersionWithNullFlags) { - int8_t null_key_flags = input.ReadByte(); - if ((null_key_flags & kFirstKeyIsNull) != 0) { - first_key.reset(); + if (format_version != kFormatVersionWithNullFlags) { + return Status::Invalid( + fmt::format("Unsupported BTree index metadata version {}.", format_version)); + } + int8_t null_key_flags = input.ReadByte(); + constexpr int8_t kKnownNullFlags = kFirstKeyIsNull | kLastKeyIsNull; + if ((null_key_flags & ~kKnownNullFlags) != 0) { + return Status::Invalid( + fmt::format("BTree index metadata has invalid null-key flags {}.", null_key_flags)); + } + if ((null_key_flags & kFirstKeyIsNull) != 0) { + if (first_key_len != 0) { + return Status::Invalid("BTree index metadata marks a non-empty first key as null."); } - if ((null_key_flags & kLastKeyIsNull) != 0) { - last_key.reset(); + first_key.reset(); + } + if ((null_key_flags & kLastKeyIsNull) != 0) { + if (last_key_len != 0) { + return Status::Invalid("BTree index metadata marks a non-empty last key as null."); } + last_key.reset(); } - } else if (first_key_len == 0 && last_key_len == 0 && has_nulls) { + } else if (input.Available() == 0 && first_key_len == 0 && last_key_len == 0 && has_nulls) { // Legacy metadata used zero length for null keys. Both empty boundaries plus a null bitmap // identify an all-null file; a single empty boundary remains a valid serialized key. first_key.reset(); last_key.reset(); + } else if (input.Available() != 0) { + return Status::Invalid(fmt::format("BTree index metadata has {} unexpected trailing bytes.", + input.Available())); } return std::make_shared(first_key, last_key, has_nulls); } diff --git a/src/paimon/common/global_index/btree/btree_index_meta.h b/src/paimon/common/global_index/btree/btree_index_meta.h index 85df5ff30..eb8158f12 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.h +++ b/src/paimon/common/global_index/btree/btree_index_meta.h @@ -24,6 +24,7 @@ #include "paimon/common/memory/memory_slice_input.h" #include "paimon/memory/bytes.h" +#include "paimon/result.h" namespace paimon { /// Index metadata for each BTree index file. @@ -31,8 +32,8 @@ namespace paimon { /// Empty serialized keys are valid, so null boundary keys are encoded separately with flags. class BTreeIndexMeta { public: - static std::shared_ptr Deserialize(const std::shared_ptr& meta, - paimon::MemoryPool* pool); + static Result> Deserialize(const std::shared_ptr& meta, + paimon::MemoryPool* pool); std::shared_ptr Serialize(paimon::MemoryPool* pool) const; public: diff --git a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp b/src/paimon/common/global_index/btree/btree_index_meta_test.cpp index 70c4dfcb4..b0d78fa3b 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp +++ b/src/paimon/common/global_index/btree/btree_index_meta_test.cpp @@ -19,9 +19,14 @@ #include "paimon/common/global_index/btree/btree_index_meta.h" +#include +#include +#include + #include "gtest/gtest.h" #include "paimon/common/memory/memory_slice_output.h" #include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" namespace paimon::test { class BTreeIndexMetaTest : public ::testing::Test { @@ -48,6 +53,28 @@ class BTreeIndexMetaTest : public ::testing::Test { return output.ToSlice().CopyBytes(pool_.get()); } + std::shared_ptr MetadataBytes(int32_t first_key_length, const std::string& first_key, + int32_t last_key_length, const std::string& last_key, + int8_t has_nulls, + const std::vector& suffix = {}) const { + MemorySliceOutput output( + static_cast(first_key.size() + last_key.size() + suffix.size() + 9), + pool_.get()); + output.WriteValue(first_key_length); + if (!first_key.empty()) { + output.WriteBytes(std::make_shared(first_key, pool_.get())); + } + output.WriteValue(last_key_length); + if (!last_key.empty()) { + output.WriteBytes(std::make_shared(last_key, pool_.get())); + } + output.WriteValue(has_nulls); + for (int8_t byte : suffix) { + output.WriteValue(byte); + } + return output.ToSlice().CopyBytes(pool_.get()); + } + std::shared_ptr pool_; }; @@ -62,7 +89,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeNormalKeys) { ASSERT_GT(serialized->size(), 0u); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key @@ -88,7 +116,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstKey) { auto serialized = meta->Serialize(pool_.get()); ASSERT_EQ(serialized->size(), 11 + last_key->size()); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -105,7 +134,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstAndLastKeysWithNulls) { auto serialized = meta->Serialize(pool_.get()); ASSERT_EQ(serialized->size(), 11); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -120,7 +150,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeOnlyNulls) { auto serialized = meta->Serialize(pool_.get()); ASSERT_EQ(serialized->size(), 11); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); ASSERT_FALSE(deserialized->FirstKey()); ASSERT_FALSE(deserialized->LastKey()); @@ -132,7 +163,8 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyOnlyNulls) { auto empty_key = std::make_shared(0, pool_.get()); auto serialized = LegacyMetaBytes(empty_key, empty_key, true); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_FALSE(deserialized->FirstKey()); ASSERT_FALSE(deserialized->LastKey()); ASSERT_TRUE(deserialized->HasNulls()); @@ -144,7 +176,8 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstKey) { auto last_key = std::make_shared("last_key_data", pool_.get()); auto serialized = LegacyMetaBytes(empty_key, last_key, false); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -158,7 +191,8 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) { auto empty_key = std::make_shared(0, pool_.get()); auto serialized = LegacyMetaBytes(empty_key, empty_key, false); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -167,6 +201,52 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) { ASSERT_FALSE(deserialized->OnlyNulls()); } +TEST_F(BTreeIndexMetaTest, DeserializeRejectsNullAndTruncatedMetadata) { + ASSERT_NOK(BTreeIndexMeta::Deserialize(nullptr, pool_.get())); + for (size_t size = 0; size < 9; size++) { + auto truncated = std::make_shared(std::string(size, '\0'), pool_.get()); + ASSERT_NOK(BTreeIndexMeta::Deserialize(truncated, pool_.get())); + } +} + +TEST_F(BTreeIndexMetaTest, DeserializeRejectsInvalidKeyLengths) { + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(/*first_key_length=*/-1, "", /*last_key_length=*/0, "", + /*has_nulls=*/0), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(/*first_key_length=*/100, "", /*last_key_length=*/0, "", + /*has_nulls=*/0), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(/*first_key_length=*/0, "", /*last_key_length=*/-1, "", + /*has_nulls=*/0), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(/*first_key_length=*/0, "", /*last_key_length=*/100, "", + /*has_nulls=*/0), + pool_.get())); +} + +TEST_F(BTreeIndexMetaTest, DeserializeRejectsMalformedFlagsAndTrailingBytes) { + ASSERT_NOK( + BTreeIndexMeta::Deserialize(MetadataBytes(0, "", 0, "", /*has_nulls=*/2), pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*truncated_version=*/1}), pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*unsupported_version=*/2, /*flags=*/0}), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*version=*/1, /*unknown_flags=*/4}), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*version=*/1, /*flags=*/0, /*trailing=*/0}), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(1, "x", 0, "", /*has_nulls=*/0, {/*version=*/1, /*first_key_is_null=*/1}), + pool_.get())); +} + TEST_F(BTreeIndexMetaTest, HasNullsAndOnlyNulls) { // Case 1: Has nulls with keys auto meta1 = @@ -204,7 +284,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeNoNulls) { ASSERT_TRUE(serialized); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify has_nulls is false @@ -221,7 +302,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyFirstKey) { ASSERT_TRUE(serialized); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key @@ -243,7 +325,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyLastKey) { ASSERT_TRUE(serialized); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key is null @@ -268,7 +351,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeBinaryKeys) { ASSERT_TRUE(serialized); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key diff --git a/src/paimon/common/global_index/btree/key_serializer.cpp b/src/paimon/common/global_index/btree/key_serializer.cpp index 1ce1f6b4c..464f37ea4 100644 --- a/src/paimon/common/global_index/btree/key_serializer.cpp +++ b/src/paimon/common/global_index/btree/key_serializer.cpp @@ -22,15 +22,107 @@ #include "fmt/format.h" #include "paimon/common/memory/memory_slice_input.h" #include "paimon/common/memory/memory_slice_output.h" +#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/field_type_utils.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/preconditions.h" +#include "paimon/common/utils/var_length_int_utils.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/status.h" namespace paimon { +namespace { + +Status ValidateExactLength(const MemorySlice& slice, const std::shared_ptr& type, + int32_t expected_length) { + if (slice.Length() != expected_length) { + return Status::Invalid( + fmt::format("Invalid serialized {} key length: expected {}, but found {}.", + type->ToString(), expected_length, slice.Length())); + } + return Status::OK(); +} + +Status ValidateNonCompactTimestamp(const MemorySlice& slice) { + constexpr int32_t kMillisLength = sizeof(int64_t); + constexpr int32_t kMinimumLength = kMillisLength + 1; + // nano-of-millisecond is at most 999,999, which uses no more than three varint bytes. + constexpr int32_t kMaximumNanosLength = 3; + constexpr int32_t kMaximumLength = kMillisLength + kMaximumNanosLength; + if (slice.Length() < kMinimumLength || slice.Length() > kMaximumLength) { + return Status::Invalid(fmt::format( + "Invalid serialized timestamp key length: expected between {} and {}, but found {}.", + kMinimumLength, kMaximumLength, slice.Length())); + } + + int32_t terminal_position = -1; + for (int32_t position = kMillisLength; position < slice.Length(); ++position) { + auto byte = static_cast(slice.Data()[position]); + if ((byte & 0x80) == 0) { + terminal_position = position; + break; + } + } + if (terminal_position == -1) { + return Status::Invalid("Serialized timestamp key contains an unterminated nanos varint."); + } + if (terminal_position != slice.Length() - 1) { + return Status::Invalid("Serialized timestamp key contains trailing bytes."); + } + + int32_t offset = kMillisLength; + PAIMON_ASSIGN_OR_RAISE(int32_t nanos, VarLengthIntUtils::DecodeInt(slice.Data(), &offset)); + if (nanos > 999999) { + return Status::Invalid( + fmt::format("Serialized timestamp key has invalid nanos value {}.", nanos)); + } + return Status::OK(); +} + +Status ValidateNonCompactDecimal(const MemorySlice& slice) { + constexpr int32_t kMaximumLength = sizeof(Decimal::int128_t); + if (slice.Length() < 1 || slice.Length() > kMaximumLength) { + return Status::Invalid(fmt::format( + "Invalid serialized decimal key length: expected between 1 and {}, but found {}.", + kMaximumLength, slice.Length())); + } + if (slice.Length() == 1) { + return Status::OK(); + } + + auto first = static_cast(slice.Data()[0]); + auto second = static_cast(slice.Data()[1]); + if ((first == 0 && (second & 0x80) == 0) || (first == 0xFF && (second & 0x80) != 0)) { + return Status::Invalid("Serialized decimal key contains redundant sign-extension bytes."); + } + return Status::OK(); +} + +Status ValidateDecimal(const MemorySlice& slice, + const std::shared_ptr& type) { + arrow::Decimal128 value; + if (Decimal::IsCompact(type->precision())) { + PAIMON_RETURN_NOT_OK(ValidateExactLength(slice, type, sizeof(int64_t))); + value = arrow::Decimal128(slice.ReadLong(0)); + } else { + PAIMON_RETURN_NOT_OK(ValidateNonCompactDecimal(slice)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + arrow::Decimal128 decoded, + arrow::Decimal128::FromBigEndian(reinterpret_cast(slice.Data()), + slice.Length())); + value = decoded; + } + if (!value.FitsInPrecision(type->precision())) { + return Status::Invalid( + fmt::format("Serialized decimal key does not fit precision {}.", type->precision())); + } + return Status::OK(); +} + +} // namespace + Result> KeySerializer::SerializeKey( const Literal& literal, const std::shared_ptr& type, MemoryPool* pool) { if (literal.IsNull()) { @@ -139,6 +231,7 @@ Result> KeySerializer::SerializeKey( Result KeySerializer::DeserializeKey(const MemorySlice& slice, const std::shared_ptr& type, MemoryPool* pool) { + PAIMON_RETURN_NOT_OK(ValidateSerializedKey(slice, type)); switch (type->id()) { case arrow::Type::type::BOOL: return Literal(slice.ReadByte(0) == 1 ? true : false); @@ -196,6 +289,51 @@ Result KeySerializer::DeserializeKey(const MemorySlice& slice, } } +Status KeySerializer::ValidateSerializedKey(const MemorySlice& slice, + const std::shared_ptr& type) { + if (type == nullptr) { + return Status::Invalid("Cannot validate a serialized BTree key without a key type."); + } + switch (type->id()) { + case arrow::Type::type::BOOL: { + PAIMON_RETURN_NOT_OK(ValidateExactLength(slice, type, sizeof(int8_t))); + auto value = static_cast(slice.Data()[0]); + if (value > 1) { + return Status::Invalid( + fmt::format("Invalid serialized boolean key value {}.", value)); + } + return Status::OK(); + } + case arrow::Type::type::INT8: + return ValidateExactLength(slice, type, sizeof(int8_t)); + case arrow::Type::type::INT16: + return ValidateExactLength(slice, type, sizeof(int16_t)); + case arrow::Type::type::INT32: + case arrow::Type::type::DATE32: + case arrow::Type::type::FLOAT: + return ValidateExactLength(slice, type, sizeof(int32_t)); + case arrow::Type::type::INT64: + case arrow::Type::type::DOUBLE: + return ValidateExactLength(slice, type, sizeof(int64_t)); + case arrow::Type::type::STRING: + return Status::OK(); + case arrow::Type::type::TIMESTAMP: { + auto timestamp_type = checked_pointer_cast(type); + if (Timestamp::IsCompact(DateTimeUtils::GetPrecisionFromType(timestamp_type))) { + return ValidateExactLength(slice, type, sizeof(int64_t)); + } + return ValidateNonCompactTimestamp(slice); + } + case arrow::Type::type::DECIMAL128: { + auto decimal_type = checked_pointer_cast(type); + return ValidateDecimal(slice, decimal_type); + } + default: + return Status::Invalid(fmt::format( + "Not support validate serialized {} type in BTreeGlobalIndex", type->ToString())); + } +} + MemorySlice::SliceComparator KeySerializer::CreateComparator( const std::shared_ptr& type, const std::shared_ptr& pool) { // Fast paths for integer and string types: direct value comparison without Literal diff --git a/src/paimon/common/global_index/btree/key_serializer.h b/src/paimon/common/global_index/btree/key_serializer.h index 82f851030..d44b5bdc1 100644 --- a/src/paimon/common/global_index/btree/key_serializer.h +++ b/src/paimon/common/global_index/btree/key_serializer.h @@ -38,6 +38,9 @@ class KeySerializer { const std::shared_ptr& type, MemoryPool* pool); + static Status ValidateSerializedKey(const MemorySlice& slice, + const std::shared_ptr& type); + static MemorySlice::SliceComparator CreateComparator( const std::shared_ptr& type, const std::shared_ptr& pool); }; diff --git a/src/paimon/common/global_index/btree/key_serializer_test.cpp b/src/paimon/common/global_index/btree/key_serializer_test.cpp index 3dc8a0f68..e36e72ed6 100644 --- a/src/paimon/common/global_index/btree/key_serializer_test.cpp +++ b/src/paimon/common/global_index/btree/key_serializer_test.cpp @@ -208,6 +208,42 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) { } } +TEST_F(KeySerializerTest, RejectsMalformedSerializedKeys) { + auto wrap = [this](const std::string& value) { + return MemorySlice::Wrap(std::make_shared(value, pool_.get())); + }; + + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(std::string(3, '\0')), arrow::int32())); + ASSERT_NOK( + KeySerializer::DeserializeKey(wrap(std::string(3, '\0')), arrow::int32(), pool_.get())); + + std::string invalid_boolean(1, static_cast(2)); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(invalid_boolean), arrow::boolean())); + + auto nanos_timestamp = arrow::timestamp(arrow::TimeUnit::NANO); + std::string unterminated_timestamp(9, '\0'); + unterminated_timestamp[8] = static_cast(0x80); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(unterminated_timestamp), nanos_timestamp)); + + std::string out_of_range_timestamp(11, '\0'); + out_of_range_timestamp[8] = static_cast(0xC0); + out_of_range_timestamp[9] = static_cast(0x84); + out_of_range_timestamp[10] = static_cast(0x3D); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(out_of_range_timestamp), nanos_timestamp)); + + auto non_compact_decimal = arrow::decimal128(25, 3); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(""), non_compact_decimal)); + ASSERT_NOK( + KeySerializer::ValidateSerializedKey(wrap(std::string(17, '\0')), non_compact_decimal)); + + auto narrow_decimal = arrow::decimal128(1, 0); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out_of_range_decimal, + KeySerializer::SerializeKey(Literal(Decimal::FromUnscaledLong(10, 1, 0)), + narrow_decimal, pool_.get())); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(MemorySlice::Wrap(out_of_range_decimal), + narrow_decimal)); +} + TEST_F(KeySerializerTest, CreateComparator) { // INT comparator { diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp index bc66e17a1..4ea7576b6 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp @@ -38,35 +38,47 @@ #include "paimon/utils/roaring_bitmap64.h" namespace paimon { -LazyFilteredBTreeReader::LazyFilteredBTreeReader( +Result> LazyFilteredBTreeReader::Create( std::optional read_buffer_size, const std::vector& files, const std::shared_ptr& key_type, const std::shared_ptr& file_reader, const std::shared_ptr& cache_manager, const std::shared_ptr& pool, - const std::shared_ptr& executor) + const std::shared_ptr& executor) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_selector, + BTreeFileMetaSelector::Create(files, key_type, pool)); + return std::shared_ptr( + new LazyFilteredBTreeReader(read_buffer_size, std::move(file_selector), key_type, + file_reader, cache_manager, pool, executor)); +} + +LazyFilteredBTreeReader::LazyFilteredBTreeReader( + std::optional read_buffer_size, std::unique_ptr file_selector, + std::shared_ptr key_type, std::shared_ptr file_reader, + std::shared_ptr cache_manager, std::shared_ptr pool, + std::shared_ptr executor) : read_buffer_size_(read_buffer_size), - pool_(pool), - file_selector_(files, key_type, pool), - key_type_(key_type), - file_reader_(file_reader), - cache_manager_(cache_manager), - executor_(executor) {} + pool_(std::move(pool)), + file_selector_(std::move(file_selector)), + key_type_(std::move(key_type)), + file_reader_(std::move(file_reader)), + cache_manager_(std::move(cache_manager)), + executor_(std::move(executor)) {} Result> LazyFilteredBTreeReader::VisitIsNotNull() { return DispatchVisit( - [this]() { return file_selector_.VisitIsNotNull(); }, + [this]() { return file_selector_->VisitIsNotNull(); }, [](const std::shared_ptr& reader) { return reader->VisitIsNotNull(); }); } Result> LazyFilteredBTreeReader::VisitIsNull() { return DispatchVisit( - [this]() { return file_selector_.VisitIsNull(); }, + [this]() { return file_selector_->VisitIsNull(); }, [](const std::shared_ptr& reader) { return reader->VisitIsNull(); }); } Result> LazyFilteredBTreeReader::VisitEqual( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitEqual(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitEqual(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitEqual(literal); }); @@ -74,7 +86,7 @@ Result> LazyFilteredBTreeReader::VisitEqual( Result> LazyFilteredBTreeReader::VisitNotEqual( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitNotEqual(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitNotEqual(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitNotEqual(literal); }); @@ -82,7 +94,7 @@ Result> LazyFilteredBTreeReader::VisitNotEqua Result> LazyFilteredBTreeReader::VisitLessThan( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitLessThan(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitLessThan(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitLessThan(literal); }); @@ -90,7 +102,7 @@ Result> LazyFilteredBTreeReader::VisitLessTha Result> LazyFilteredBTreeReader::VisitLessOrEqual( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitLessOrEqual(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitLessOrEqual(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitLessOrEqual(literal); }); @@ -98,7 +110,7 @@ Result> LazyFilteredBTreeReader::VisitLessOrE Result> LazyFilteredBTreeReader::VisitGreaterThan( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitGreaterThan(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitGreaterThan(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitGreaterThan(literal); }); @@ -106,15 +118,16 @@ Result> LazyFilteredBTreeReader::VisitGreater Result> LazyFilteredBTreeReader::VisitGreaterOrEqual( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitGreaterOrEqual(literal); }, - [&literal](const std::shared_ptr& reader) { - return reader->VisitGreaterOrEqual(literal); - }); + return DispatchVisit( + [this, &literal]() { return file_selector_->VisitGreaterOrEqual(literal); }, + [&literal](const std::shared_ptr& reader) { + return reader->VisitGreaterOrEqual(literal); + }); } Result> LazyFilteredBTreeReader::VisitIn( const std::vector& literals) { - return DispatchVisit([this, &literals]() { return file_selector_.VisitIn(literals); }, + return DispatchVisit([this, &literals]() { return file_selector_->VisitIn(literals); }, [&literals](const std::shared_ptr& reader) { return reader->VisitIn(literals); }); @@ -122,7 +135,7 @@ Result> LazyFilteredBTreeReader::VisitIn( Result> LazyFilteredBTreeReader::VisitNotIn( const std::vector& literals) { - return DispatchVisit([this, &literals]() { return file_selector_.VisitNotIn(literals); }, + return DispatchVisit([this, &literals]() { return file_selector_->VisitNotIn(literals); }, [&literals](const std::shared_ptr& reader) { return reader->VisitNotIn(literals); }); @@ -130,7 +143,7 @@ Result> LazyFilteredBTreeReader::VisitNotIn( Result> LazyFilteredBTreeReader::VisitStartsWith( const Literal& prefix) { - return DispatchVisit([this, &prefix]() { return file_selector_.VisitStartsWith(prefix); }, + return DispatchVisit([this, &prefix]() { return file_selector_->VisitStartsWith(prefix); }, [&prefix](const std::shared_ptr& reader) { return reader->VisitStartsWith(prefix); }); @@ -138,7 +151,7 @@ Result> LazyFilteredBTreeReader::VisitStartsW Result> LazyFilteredBTreeReader::VisitEndsWith( const Literal& suffix) { - return DispatchVisit([this, &suffix]() { return file_selector_.VisitEndsWith(suffix); }, + return DispatchVisit([this, &suffix]() { return file_selector_->VisitEndsWith(suffix); }, [&suffix](const std::shared_ptr& reader) { return reader->VisitEndsWith(suffix); }); @@ -146,7 +159,7 @@ Result> LazyFilteredBTreeReader::VisitEndsWit Result> LazyFilteredBTreeReader::VisitContains( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitContains(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitContains(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitContains(literal); }); @@ -154,7 +167,7 @@ Result> LazyFilteredBTreeReader::VisitContain Result> LazyFilteredBTreeReader::VisitLike( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitLike(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitLike(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitLike(literal); }); @@ -214,7 +227,8 @@ Result> LazyFilteredBTreeReader::CreateSingle auto comparator = KeySerializer::CreateComparator(key_type_, pool_); // Get min/max key slices from meta data (keep as slices; Create() will deserialize) - auto index_meta = BTreeIndexMeta::Deserialize(meta.metadata, pool_.get()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_meta, + BTreeIndexMeta::Deserialize(meta.metadata, pool_.get())); std::optional min_key_slice; std::optional max_key_slice; if (index_meta->FirstKey()) { diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h index 0603e78cf..74ed978b8 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h @@ -40,13 +40,12 @@ namespace paimon { class LazyFilteredBTreeReader : public GlobalIndexReader { public: - LazyFilteredBTreeReader(std::optional read_buffer_size, - const std::vector& files, - const std::shared_ptr& key_type, - const std::shared_ptr& file_reader, - const std::shared_ptr& cache_manager, - const std::shared_ptr& pool, - const std::shared_ptr& executor); + static Result> Create( + std::optional read_buffer_size, const std::vector& files, + const std::shared_ptr& key_type, + const std::shared_ptr& file_reader, + const std::shared_ptr& cache_manager, const std::shared_ptr& pool, + const std::shared_ptr& executor); Result> VisitIsNotNull() override; Result> VisitIsNull() override; @@ -80,6 +79,13 @@ class LazyFilteredBTreeReader : public GlobalIndexReader { } private: + LazyFilteredBTreeReader(std::optional read_buffer_size, + std::unique_ptr file_selector, + std::shared_ptr key_type, + std::shared_ptr file_reader, + std::shared_ptr cache_manager, + std::shared_ptr pool, std::shared_ptr executor); + using SelectAction = std::function>()>; using ReaderAction = std::function>( const std::shared_ptr&)>; @@ -96,7 +102,7 @@ class LazyFilteredBTreeReader : public GlobalIndexReader { private: std::optional read_buffer_size_; std::shared_ptr pool_; - BTreeFileMetaSelector file_selector_; + std::unique_ptr file_selector_; std::shared_ptr key_type_; std::shared_ptr file_reader_; std::shared_ptr cache_manager_; diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp index bf5503fa7..a1be0108b 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp @@ -139,9 +139,11 @@ class LazyFilteredBTreeReaderTest : public ::testing::Test { const std::shared_ptr& executor = nullptr) const { auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); - return std::make_shared(/*read_buffer_size=*/std::nullopt, - all_metas_, arrow::int32(), file_reader, - cache_manager, pool_, executor); + EXPECT_OK_AND_ASSIGN(std::shared_ptr reader, + LazyFilteredBTreeReader::Create( + /*read_buffer_size=*/std::nullopt, all_metas_, arrow::int32(), + file_reader, cache_manager, pool_, executor)); + return reader; } void CheckResult(const std::shared_ptr& result, @@ -299,6 +301,23 @@ TEST_F(LazyFilteredBTreeReaderTest, TestVisitNotIn) { CheckResult(result, {3, 4, 6, 7, 10}); } +TEST_F(LazyFilteredBTreeReaderTest, RejectsMismatchedLiteralEncoding) { + auto reader = CreateReader(); + Literal int8_literal(static_cast(1)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr not_equal, + reader->VisitNotEqual(int8_literal)); + auto not_equal_bitmap = std::dynamic_pointer_cast(not_equal); + ASSERT_TRUE(not_equal_bitmap != nullptr); + ASSERT_NOK(not_equal_bitmap->GetBitmap()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr not_in, + reader->VisitNotIn({int8_literal})); + auto not_in_bitmap = std::dynamic_pointer_cast(not_in); + ASSERT_TRUE(not_in_bitmap != nullptr); + ASSERT_NOK(not_in_bitmap->GetBitmap()); +} + // --- VisitIsNull --- TEST_F(LazyFilteredBTreeReaderTest, TestVisitIsNull) { @@ -364,9 +383,11 @@ TEST_F(LazyFilteredBTreeReaderTest, TestEmptyFilesList) { std::vector empty_metas; auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); - auto reader = std::make_shared( - /*read_buffer_size=*/std::nullopt, empty_metas, arrow::int32(), file_reader, cache_manager, - pool_, /*executor=*/nullptr); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr reader, + LazyFilteredBTreeReader::Create(/*read_buffer_size=*/std::nullopt, empty_metas, + arrow::int32(), file_reader, cache_manager, pool_, + /*executor=*/nullptr)); // Any query on empty files should return empty bitmap Literal literal_1(1); @@ -456,12 +477,23 @@ TEST_F(LazyFilteredBTreeReaderTest, TestParallelEmptyFilesList) { std::vector empty_metas; auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); - auto reader = std::make_shared( - /*read_buffer_size=*/std::nullopt, empty_metas, arrow::int32(), file_reader, cache_manager, - pool_, executor); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + LazyFilteredBTreeReader::Create(/*read_buffer_size=*/std::nullopt, + empty_metas, arrow::int32(), file_reader, + cache_manager, pool_, executor)); Literal literal_1(1); ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(literal_1)); CheckEmpty(result); } +TEST_F(LazyFilteredBTreeReaderTest, RejectsMalformedFileMetadata) { + auto file_reader = std::make_shared(fs_, base_path_); + auto cache_manager = std::make_shared(1024 * 1024, 0.5); + std::vector invalid_metas = { + GlobalIndexIOMeta("invalid", 1, /*metadata=*/nullptr)}; + ASSERT_NOK(LazyFilteredBTreeReader::Create( + /*read_buffer_size=*/std::nullopt, invalid_metas, arrow::int32(), file_reader, + cache_manager, pool_, /*executor=*/nullptr)); +} + } // namespace paimon::test diff --git a/src/paimon/common/global_index/union_global_index_reader_test.cpp b/src/paimon/common/global_index/union_global_index_reader_test.cpp index 6c3c68585..721255264 100644 --- a/src/paimon/common/global_index/union_global_index_reader_test.cpp +++ b/src/paimon/common/global_index/union_global_index_reader_test.cpp @@ -220,6 +220,10 @@ class DeferAfterFirstExecutor : public Executor { return 1; } + uint32_t SubmissionCount() const { + return submission_count_; + } + void RunPendingTasks() { while (!pending_tasks_.empty()) { std::function task = std::move(pending_tasks_.front()); @@ -265,13 +269,15 @@ class UnionGlobalIndexReaderTest : public ::testing::Test { TEST_F(UnionGlobalIndexReaderTest, TestSingleReaderUnion) { auto reader = std::make_shared(); reader->SetDefaultResult({1, 2, 3}); + auto executor = std::make_shared(); std::vector> readers = {reader}; - UnionGlobalIndexReader union_reader(std::move(readers), nullptr); + UnionGlobalIndexReader union_reader(std::move(readers), executor); ASSERT_OK_AND_ASSIGN(auto result, union_reader.VisitIsNotNull()); CheckResult(result, {1, 2, 3}); ASSERT_EQ(reader->InvocationCount(), 1); + ASSERT_EQ(executor->SubmissionCount(), 0); } TEST_F(UnionGlobalIndexReaderTest, TestMultipleReadersUnionSequential) { diff --git a/src/paimon/core/global_index/global_index_evaluator_impl.cpp b/src/paimon/core/global_index/global_index_evaluator_impl.cpp index 5c1938153..c857a9f0b 100644 --- a/src/paimon/core/global_index/global_index_evaluator_impl.cpp +++ b/src/paimon/core/global_index/global_index_evaluator_impl.cpp @@ -19,21 +19,124 @@ #include "paimon/core/global_index/global_index_evaluator_impl.h" +#include +#include + #include "fmt/format.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/predicate_utils.h" namespace paimon { +namespace { +void FlattenChildren(const std::shared_ptr& compound_predicate, + std::vector>* flattened) { + for (const std::shared_ptr& child : compound_predicate->Children()) { + auto compound_child = std::dynamic_pointer_cast(child); + if (compound_child != nullptr && compound_child->GetFunction().GetType() == + compound_predicate->GetFunction().GetType()) { + FlattenChildren(compound_child, flattened); + } else { + flattened->push_back(child); + } + } +} + +/// A predicate is null-rejecting when it cannot match a row whose tested field is null. +/// Under SQL three-valued logic every comparison and match predicate rejects null; only +/// IS NULL accepts it, and IS NOT NULL is the predicate being pruned. +bool IsNullRejecting(const std::shared_ptr& predicate) { + auto leaf_predicate = std::dynamic_pointer_cast(predicate); + if (leaf_predicate == nullptr) { + return false; + } + switch (leaf_predicate->GetFunction().GetType()) { + case Function::Type::EQUAL: + case Function::Type::NOT_EQUAL: + case Function::Type::GREATER_THAN: + case Function::Type::GREATER_OR_EQUAL: + case Function::Type::LESS_THAN: + case Function::Type::LESS_OR_EQUAL: + case Function::Type::IN: + case Function::Type::NOT_IN: + case Function::Type::STARTS_WITH: + case Function::Type::ENDS_WITH: + case Function::Type::CONTAINS: + case Function::Type::LIKE: + return true; + default: + return false; + } +} + +bool IsIsNotNull(const std::shared_ptr& predicate) { + auto leaf_predicate = std::dynamic_pointer_cast(predicate); + return leaf_predicate != nullptr && + leaf_predicate->GetFunction().GetType() == Function::Type::IS_NOT_NULL; +} +} // namespace + Result> GlobalIndexEvaluatorImpl::Evaluate( const std::shared_ptr& predicate) { std::shared_ptr compound_result; if (predicate) { - PAIMON_ASSIGN_OR_RAISE(compound_result, EvaluatePredicate(predicate)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_predicate, + NormalizePredicate(predicate)); + PAIMON_ASSIGN_OR_RAISE(compound_result, EvaluatePredicate(normalized_predicate)); } return compound_result; } +Result> GlobalIndexEvaluatorImpl::NormalizePredicate( + const std::shared_ptr& predicate) { + auto compound_predicate = std::dynamic_pointer_cast(predicate); + if (compound_predicate == nullptr) { + return predicate; + } + std::vector> children; + FlattenChildren(compound_predicate, &children); + + bool is_and = compound_predicate->GetFunction().GetType() == Function::Type::AND; + if (is_and) { + std::set constrained_fields; + for (const std::shared_ptr& child : children) { + auto leaf = std::dynamic_pointer_cast(child); + if (leaf != nullptr && IsNullRejecting(child)) { + constrained_fields.insert(leaf->FieldName()); + } + } + if (!constrained_fields.empty()) { + std::vector> pruned; + pruned.reserve(children.size()); + for (const std::shared_ptr& child : children) { + auto leaf = std::dynamic_pointer_cast(child); + if (leaf != nullptr && IsIsNotNull(child) && + constrained_fields.count(leaf->FieldName()) > 0) { + continue; + } + pruned.push_back(child); + } + children = std::move(pruned); + } + } + + std::vector> normalized_children; + normalized_children.reserve(children.size()); + for (const std::shared_ptr& child : children) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_child, + NormalizePredicate(child)); + normalized_children.push_back(std::move(normalized_child)); + } + if (normalized_children.size() == 1) { + return normalized_children[0]; + } + if (is_and) { + return PredicateBuilder::And(normalized_children); + } + return PredicateBuilder::Or(normalized_children); +} + Result>> GlobalIndexEvaluatorImpl::GetIndexReaders( const std::string& field_name) { PAIMON_ASSIGN_OR_RAISE(DataField data_field, table_schema_->GetField(field_name)); diff --git a/src/paimon/core/global_index/global_index_evaluator_impl.h b/src/paimon/core/global_index/global_index_evaluator_impl.h index 7555a716e..7c1591870 100644 --- a/src/paimon/core/global_index/global_index_evaluator_impl.h +++ b/src/paimon/core/global_index/global_index_evaluator_impl.h @@ -46,6 +46,11 @@ class GlobalIndexEvaluatorImpl : public GlobalIndexEvaluator { Result> Evaluate( const std::shared_ptr& predicate) override; + /// Applies Java-compatible compound flattening and removes redundant IS NOT NULL + /// predicates from AND expressions. + static Result> NormalizePredicate( + const std::shared_ptr& predicate); + private: Result> EvaluatePredicate( const std::shared_ptr& predicate); diff --git a/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp b/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp new file mode 100644 index 000000000..46253cf23 --- /dev/null +++ b/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/global_index/global_index_evaluator_impl.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { +class RecordingGlobalIndexReader : public GlobalIndexReader { + public: + Result> VisitIsNotNull() override { + is_not_null_calls_++; + return Bitmap({0, 1}); + } + + Result> VisitIsNull() override { + is_null_calls_++; + return Bitmap({2}); + } + + Result> VisitEqual(const Literal& literal) override { + equal_calls_++; + return Bitmap({1, 3}); + } + + Result> VisitNotEqual(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitLessThan(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitLessOrEqual(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitGreaterThan(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitGreaterOrEqual( + const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitIn( + const std::vector& literals) override { + return NotEvaluable(); + } + + Result> VisitNotIn( + const std::vector& literals) override { + return NotEvaluable(); + } + + Result> VisitStartsWith(const Literal& prefix) override { + return NotEvaluable(); + } + + Result> VisitEndsWith(const Literal& suffix) override { + return NotEvaluable(); + } + + Result> VisitContains(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitLike(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitVectorSearch( + const std::shared_ptr& vector_search) override { + return Status::Invalid("not supported"); + } + + Result> VisitFullTextSearch( + const std::shared_ptr& full_text_search) override { + return Status::Invalid("not supported"); + } + + bool IsThreadSafe() const override { + return false; + } + + std::string GetIndexType() const override { + return "test"; + } + + int32_t EqualCalls() const { + return equal_calls_; + } + + int32_t IsNullCalls() const { + return is_null_calls_; + } + + int32_t IsNotNullCalls() const { + return is_not_null_calls_; + } + + private: + static std::shared_ptr Bitmap(std::initializer_list positions) { + RoaringBitmap64 bitmap; + for (int64_t position : positions) { + bitmap.Add(position); + } + return std::make_shared( + [bitmap = std::move(bitmap)]() -> Result { return bitmap; }); + } + + static std::shared_ptr NotEvaluable() { + return nullptr; + } + + int32_t equal_calls_ = 0; + int32_t is_null_calls_ = 0; + int32_t is_not_null_calls_ = 0; +}; + +std::set CollectPositions(const std::shared_ptr& result) { + std::set positions; + EXPECT_TRUE(result != nullptr); + if (result == nullptr) { + return positions; + } + EXPECT_OK_AND_ASSIGN(std::unique_ptr iterator, + result->CreateIterator()); + while (iterator->HasNext()) { + positions.insert(iterator->Next()); + } + return positions; +} +} // namespace + +class GlobalIndexEvaluatorImplTest : public ::testing::Test { + protected: + void SetUp() override { + std::vector fields = { + DataField(0, arrow::field("a", arrow::int64())), + DataField(1, arrow::field("b", arrow::int64())), + }; + table_schema_ = std::make_shared( + /*version=*/1, /*id=*/0, fields, /*highest_field_id=*/1, + /*partition_keys=*/std::vector(), + /*primary_keys=*/std::vector(), + /*options=*/std::map(), /*comment=*/std::nullopt, + /*time_millis=*/0); + reader_ = std::make_shared(); + } + + GlobalIndexEvaluatorImpl CreateEvaluator() const { + std::shared_ptr reader = reader_; + return GlobalIndexEvaluatorImpl( + table_schema_, + [reader](int32_t field_id) -> Result>> { + if (field_id == 0) { + return std::vector>{reader}; + } + return std::vector>(); + }); + } + + std::shared_ptr Equal(const std::string& field_name, int32_t field_index) const { + return PredicateBuilder::Equal(field_index, field_name, FieldType::BIGINT, + Literal(static_cast(42))); + } + + std::shared_ptr table_schema_; + std::shared_ptr reader_; +}; + +TEST_F(GlobalIndexEvaluatorImplTest, SupportedAndUnsupportedLeavesCombineSafely) { + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + std::shared_ptr indexed = Equal("a", 0); + std::shared_ptr unindexed = Equal("b", 1); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr leaf_result, + evaluator.Evaluate(indexed)); + ASSERT_EQ((std::set{1, 3}), CollectPositions(leaf_result)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr unsupported_result, + evaluator.Evaluate(unindexed)); + ASSERT_EQ(nullptr, unsupported_result); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_predicate, + PredicateBuilder::And({indexed, unindexed})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_result, + evaluator.Evaluate(and_predicate)); + ASSERT_EQ((std::set{1, 3}), CollectPositions(and_result)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_predicate, + PredicateBuilder::Or({indexed, unindexed})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_result, + evaluator.Evaluate(or_predicate)); + ASSERT_EQ(nullptr, or_result); +} + +TEST_F(GlobalIndexEvaluatorImplTest, NormalizationFlattensNestedCompounds) { + std::shared_ptr first = Equal("a", 0); + std::shared_ptr second = PredicateBuilder::IsNull(0, "a", FieldType::BIGINT); + std::shared_ptr third = Equal("b", 1); + ASSERT_OK_AND_ASSIGN(std::shared_ptr nested, PredicateBuilder::And({first, second})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({nested, third})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr normalized, + GlobalIndexEvaluatorImpl::NormalizePredicate(predicate)); + auto compound = std::dynamic_pointer_cast(normalized); + ASSERT_TRUE(compound != nullptr); + ASSERT_EQ(Function::Type::AND, compound->GetFunction().GetType()); + ASSERT_EQ(3, compound->Children().size()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, RedundantIsNotNullIsPrunedFromAnd) { + std::shared_ptr equal = Equal("a", 0); + std::shared_ptr is_not_null = PredicateBuilder::IsNotNull(0, "a", FieldType::BIGINT); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({equal, is_not_null})); + + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator.Evaluate(predicate)); + ASSERT_EQ((std::set{1, 3}), CollectPositions(result)); + ASSERT_EQ(1, reader_->EqualCalls()); + ASSERT_EQ(0, reader_->IsNotNullCalls()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, IsNullDoesNotMakeIsNotNullRedundant) { + std::shared_ptr is_null = PredicateBuilder::IsNull(0, "a", FieldType::BIGINT); + std::shared_ptr is_not_null = PredicateBuilder::IsNotNull(0, "a", FieldType::BIGINT); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({is_null, is_not_null})); + + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator.Evaluate(predicate)); + ASSERT_TRUE(CollectPositions(result).empty()); + ASSERT_EQ(1, reader_->IsNullCalls()); + ASSERT_EQ(1, reader_->IsNotNullCalls()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, OrDoesNotPruneIsNotNull) { + std::shared_ptr equal = Equal("a", 0); + std::shared_ptr is_not_null = PredicateBuilder::IsNotNull(0, "a", FieldType::BIGINT); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::Or({equal, is_not_null})); + + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator.Evaluate(predicate)); + ASSERT_EQ((std::set{0, 1, 3}), CollectPositions(result)); + ASSERT_EQ(1, reader_->EqualCalls()); + ASSERT_EQ(1, reader_->IsNotNullCalls()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, NullPredicateIsNotEvaluated) { + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + evaluator.Evaluate(/*predicate=*/nullptr)); + ASSERT_EQ(nullptr, result); +} + +} // namespace paimon::test diff --git a/src/paimon/core/index/pk/primary_key_index_definition.h b/src/paimon/core/index/pk/primary_key_index_definition.h new file mode 100644 index 000000000..1a06628e9 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_definition.h @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace paimon { +/// Resolved definition of one source-backed primary-key index. +class PrimaryKeyIndexDefinition { + public: + /// Built-in primary-key index families. + enum class Family { + VECTOR, + BTREE, + BITMAP, + FULL_TEXT, + }; + + PrimaryKeyIndexDefinition(std::string column, int32_t field_id, std::string index_type, + Family family, std::map options) + : column_(std::move(column)), + field_id_(field_id), + index_type_(std::move(index_type)), + family_(family), + options_(std::move(options)) {} + + const std::string& Column() const { + return column_; + } + + int32_t FieldId() const { + return field_id_; + } + + const std::string& IndexType() const { + return index_type_; + } + + Family GetFamily() const { + return family_; + } + + const std::map& Options() const { + return options_; + } + + private: + std::string column_; + int32_t field_id_; + std::string index_type_; + Family family_; + std::map options_; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.cpp b/src/paimon/core/index/pk/primary_key_index_definitions.cpp new file mode 100644 index 000000000..214338f4b --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_definitions.cpp @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/index/pk/primary_key_index_definitions.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/object_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/defs.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace paimon { +namespace { +using IndexOptions = std::map; + +constexpr char kBTreeIndexType[] = "btree"; +constexpr char kBitmapIndexType[] = "bitmap"; +constexpr char kFullTextIndexType[] = "full-text"; +constexpr char kBTreeOptionFamily[] = "pk-btree"; +constexpr char kBitmapOptionFamily[] = "pk-bitmap"; +constexpr char kBTreeAlgorithmPrefix[] = "btree-index."; +constexpr char kBitmapAlgorithmPrefix[] = "bitmap-index."; +constexpr char kFieldScopedPrefix[] = "fields."; +constexpr char kRecordsPerRangeKey[] = "sorted-index.records-per-range"; + +std::vector IndexColumns(const std::map& options, + const char* option_key) { + auto iter = options.find(option_key); + if (iter == options.end()) { + return {}; + } + std::vector columns = StringUtils::Split(iter->second, ",", false); + for (std::string& column : columns) { + StringUtils::Trim(&column); + } + return columns; +} + +Status AddUniqueColumns(const std::vector& columns, + const std::function& on_duplicate, + std::set* unique_columns) { + for (const std::string& column : columns) { + if (!unique_columns->insert(column).second) { + return on_duplicate(column); + } + } + return Status::OK(); +} + +Status ValidateNoDuplicates(const std::vector& columns, const char* option_key) { + std::set unique_columns; + return AddUniqueColumns( + columns, + [option_key](const std::string& column) { + return Status::Invalid( + fmt::format("{} contains duplicate column '{}'.", option_key, column)); + }, + &unique_columns); +} + +Status ValidateUniqueColumns(const std::vector& columns, + std::set* indexed_columns) { + return AddUniqueColumns( + columns, + [](const std::string& column) { + return Status::Invalid( + fmt::format("Column '{}' can own at most one primary-key index.", column)); + }, + indexed_columns); +} + +/// Resolves the effective option map of one sorted-index definition: table options first, +/// then the field-scoped JSON options with unqualified keys prefixed by the algorithm +/// prefix, mirroring Java `CoreOptions#primaryKeySortedIndexOptions`. +Result> SortedIndexOptions( + const std::map& table_options, const std::string& column, + const char* option_family, const char* algorithm_prefix) { + std::map resolved = table_options; + resolved.erase(kRecordsPerRangeKey); + std::string option_key = + fmt::format("{}{}.{}.index.options", kFieldScopedPrefix, column, option_family); + auto iter = table_options.find(option_key); + if (iter == table_options.end() || StringUtils::IsNullOrWhitespaceOnly(iter->second)) { + return resolved; + } + + rapidjson::Document document; + document.Parse(iter->second.c_str()); + if (document.HasParseError() || !document.IsObject()) { + return Status::Invalid( + fmt::format("{} must be a JSON object of option key-value pairs.", option_key)); + } + for (auto member = document.MemberBegin(); member != document.MemberEnd(); ++member) { + if (!member->name.IsString() || + StringUtils::IsNullOrWhitespaceOnly(member->name.GetString())) { + return Status::Invalid(fmt::format("{} contains an empty option key.", option_key)); + } + std::string key = member->name.GetString(); + if (member->value.IsNull()) { + return Status::Invalid( + fmt::format("{} value for key {} must not be null.", option_key, key)); + } + if (member->value.IsObject() || member->value.IsArray()) { + return Status::Invalid( + fmt::format("{} must be a JSON object of option key-value pairs.", option_key)); + } + std::string value; + if (member->value.IsString()) { + value = member->value.GetString(); + } else { + // Java's parseJsonMap(..., String.class) coerces scalar JSON values (numbers, + // booleans) to their text form, so `{"compression-level":3}` is valid there. + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + member->value.Accept(writer); + value = buffer.GetString(); + } + std::string qualified_key = StringUtils::StartsWith(key, algorithm_prefix) || + StringUtils::StartsWith(key, kFieldScopedPrefix) + ? key + : algorithm_prefix + key; + auto previous = resolved.find(qualified_key); + if (previous != resolved.end() && previous->second != value) { + return Status::Invalid( + fmt::format("{} defines conflicting values for {}.", option_key, qualified_key)); + } + resolved[qualified_key] = value; + } + return resolved; +} + +} // namespace + +Result PrimaryKeyIndexDefinitions::Create(const TableSchema& schema) { + const std::map& options = schema.Options(); + std::vector vector_columns = + IndexColumns(options, Options::PK_VECTOR_INDEX_COLUMNS); + std::vector btree_columns = IndexColumns(options, Options::PK_BTREE_INDEX_COLUMNS); + std::vector bitmap_columns = + IndexColumns(options, Options::PK_BITMAP_INDEX_COLUMNS); + std::vector full_text_columns = + IndexColumns(options, Options::PK_FULL_TEXT_INDEX_COLUMNS); + PAIMON_RETURN_NOT_OK(ValidateNoDuplicates(vector_columns, Options::PK_VECTOR_INDEX_COLUMNS)); + PAIMON_RETURN_NOT_OK(ValidateNoDuplicates(btree_columns, Options::PK_BTREE_INDEX_COLUMNS)); + PAIMON_RETURN_NOT_OK(ValidateNoDuplicates(bitmap_columns, Options::PK_BITMAP_INDEX_COLUMNS)); + PAIMON_RETURN_NOT_OK( + ValidateNoDuplicates(full_text_columns, Options::PK_FULL_TEXT_INDEX_COLUMNS)); + std::set indexed_columns; + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(vector_columns, &indexed_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(btree_columns, &indexed_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(bitmap_columns, &indexed_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(full_text_columns, &indexed_columns)); + + std::vector definitions; + for (const DataField& field : schema.Fields()) { + const std::string& column = field.Name(); + if (ObjectUtils::Contains(btree_columns, column)) { + PAIMON_ASSIGN_OR_RAISE( + IndexOptions definition_options, + SortedIndexOptions(options, column, kBTreeOptionFamily, kBTreeAlgorithmPrefix)); + definitions.emplace_back(column, field.Id(), kBTreeIndexType, + PrimaryKeyIndexDefinition::Family::BTREE, + std::move(definition_options)); + } else if (ObjectUtils::Contains(bitmap_columns, column)) { + PAIMON_ASSIGN_OR_RAISE( + IndexOptions definition_options, + SortedIndexOptions(options, column, kBitmapOptionFamily, kBitmapAlgorithmPrefix)); + definitions.emplace_back(column, field.Id(), kBitmapIndexType, + PrimaryKeyIndexDefinition::Family::BITMAP, + std::move(definition_options)); + } else if (ObjectUtils::Contains(vector_columns, column)) { + std::string index_type; + auto type_iter = + options.find(fmt::format("{}{}.pk-vector.index.type", kFieldScopedPrefix, column)); + if (type_iter != options.end()) { + index_type = type_iter->second; + } + definitions.emplace_back(column, field.Id(), index_type, + PrimaryKeyIndexDefinition::Family::VECTOR, + std::map()); + } else if (ObjectUtils::Contains(full_text_columns, column)) { + definitions.emplace_back(column, field.Id(), kFullTextIndexType, + PrimaryKeyIndexDefinition::Family::FULL_TEXT, + std::map()); + } + } + return PrimaryKeyIndexDefinitions(std::move(definitions)); +} + +std::vector PrimaryKeyIndexDefinitions::ScalarDefinitions() const { + return ScalarDefinitions(definitions_); +} + +std::vector PrimaryKeyIndexDefinitions::ScalarDefinitions( + const std::vector& definitions) { + std::vector scalar_definitions; + for (const PrimaryKeyIndexDefinition& definition : definitions) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE || + definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BITMAP) { + scalar_definitions.push_back(definition); + } + } + return scalar_definitions; +} + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.h b/src/paimon/core/index/pk/primary_key_index_definitions.h new file mode 100644 index 000000000..247c6b5f4 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_definitions.h @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/index/pk/primary_key_index_definition.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/result.h" + +namespace paimon { +/// Resolves all configured source-backed primary-key indexes of a table schema. +class PrimaryKeyIndexDefinitions { + public: + /// Resolves index definitions from `pk-btree.index.columns`, `pk-bitmap.index.columns`, + /// `pk-vector.index.columns` and `pk-full-text.index.columns` together with their + /// field-scoped option JSON, rejecting duplicate columns and columns owned by more than + /// one index family. + static Result Create(const TableSchema& schema); + + const std::vector& Definitions() const { + return definitions_; + } + + /// @return The scalar (BTree / Bitmap) definitions usable by batch scans. + std::vector ScalarDefinitions() const; + + /// Filters an arbitrary definition list to the scalar families usable by batch scans. + static std::vector ScalarDefinitions( + const std::vector& definitions); + + private: + explicit PrimaryKeyIndexDefinitions(std::vector definitions) + : definitions_(std::move(definitions)) {} + + std::vector definitions_; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp b/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp new file mode 100644 index 000000000..616b0dc98 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/index/pk/primary_key_index_definitions.h" + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/index/pk/primary_key_index_definition.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/defs.h" +#include "paimon/result.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { +/// Builds a primary-key table schema with fields id BIGINT (pk), price DOUBLE, age INT, +/// status STRING and emb FLOAT, merging the given options over a fixed bucket option. +Result> MakeSchema(std::map options) { + std::vector fields = { + DataField(0, arrow::field("id", arrow::int64(), /*nullable=*/false)), + DataField(1, arrow::field("price", arrow::float64())), + DataField(2, arrow::field("age", arrow::int32())), + DataField(3, arrow::field("status", arrow::utf8())), + DataField(4, arrow::field("emb", arrow::float32()))}; + options.emplace(Options::BUCKET, "1"); + return TableSchema::Create(/*schema_id=*/0, DataField::ConvertDataFieldsToArrowSchema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options); +} +} // namespace + +TEST(PrimaryKeyIndexDefinitionsTest, NoIndexOptionsYieldsEmptyDefinitions) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, MakeSchema({})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_TRUE(definitions.Definitions().empty()); + ASSERT_TRUE(definitions.ScalarDefinitions().empty()); +} + +TEST(PrimaryKeyIndexDefinitionsTest, ResolvesBTreeDefinitions) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price,age"}})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_EQ(2, definitions.Definitions().size()); + const PrimaryKeyIndexDefinition& price = definitions.Definitions()[0]; + ASSERT_EQ("price", price.Column()); + ASSERT_EQ(1, price.FieldId()); + ASSERT_EQ("btree", price.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::BTREE, price.GetFamily()); + const PrimaryKeyIndexDefinition& age = definitions.Definitions()[1]; + ASSERT_EQ("age", age.Column()); + ASSERT_EQ(2, age.FieldId()); + ASSERT_EQ("btree", age.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::BTREE, age.GetFamily()); + ASSERT_EQ(2, definitions.ScalarDefinitions().size()); +} + +TEST(PrimaryKeyIndexDefinitionsTest, ResolvesBitmapDefinition) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BITMAP_INDEX_COLUMNS, "status"}})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_EQ(1, definitions.Definitions().size()); + const PrimaryKeyIndexDefinition& status = definitions.Definitions()[0]; + ASSERT_EQ("status", status.Column()); + ASSERT_EQ(3, status.FieldId()); + ASSERT_EQ("bitmap", status.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::BITMAP, status.GetFamily()); + ASSERT_EQ(1, definitions.ScalarDefinitions().size()); +} + +TEST(PrimaryKeyIndexDefinitionsTest, IgnoresColumnAbsentFromSchema) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "not_in_schema"}})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_TRUE(definitions.Definitions().empty()); +} + +TEST(PrimaryKeyIndexDefinitionsTest, CoercesScalarJsonOptionValuesLikeJava) { + // Java's parseJsonMap(..., String.class) accepts scalar JSON values and coerces them + // to text, so numeric or boolean values written by a Java engine must stay readable. + std::map options = { + {Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", + R"({"compression-level":3,"cache-enabled":true,"block-size":"64 kb"})"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, MakeSchema(options)); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + const std::map& resolved = definitions.Definitions()[0].Options(); + ASSERT_EQ("3", resolved.at("btree-index.compression-level")); + ASSERT_EQ("true", resolved.at("btree-index.cache-enabled")); + ASSERT_EQ("64 kb", resolved.at("btree-index.block-size")); + + // Null and nested values are rejected like in Java. + std::map null_options = { + {Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", R"({"block-size":null})"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr null_schema, MakeSchema(null_options)); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*null_schema)); + std::map nested_options = { + {Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", R"({"block-size":{"v":"64 kb"}})"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr nested_schema, MakeSchema(nested_options)); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*nested_schema)); +} + +TEST(PrimaryKeyIndexDefinitionsTest, QualifiesFieldScopedJsonOptions) { + std::map options = { + {Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"sorted-index.records-per-range", "4096"}, + {"fields.price.pk-btree.index.options", + R"({"block-size":"64 kb","btree-index.cache-size":"32 mb","fields.foo.x":"y"})"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, MakeSchema(options)); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_EQ(1, definitions.Definitions().size()); + const std::map& resolved = definitions.Definitions()[0].Options(); + // Unqualified keys are prefixed with the algorithm prefix, qualified keys are kept as-is. + ASSERT_EQ(1, resolved.count("btree-index.block-size")); + ASSERT_EQ("64 kb", resolved.at("btree-index.block-size")); + ASSERT_EQ(1, resolved.count("btree-index.cache-size")); + ASSERT_EQ("32 mb", resolved.at("btree-index.cache-size")); + ASSERT_EQ(1, resolved.count("fields.foo.x")); + ASSERT_EQ("y", resolved.at("fields.foo.x")); + // The per-range knob never leaks into the definition, other table options are retained. + ASSERT_EQ(0, resolved.count("sorted-index.records-per-range")); + ASSERT_EQ(1, resolved.count(Options::BUCKET)); + ASSERT_EQ("1", resolved.at(Options::BUCKET)); +} + +TEST(PrimaryKeyIndexDefinitionsTest, RejectsConflictingJsonOptionValue) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"btree-index.block-size", "128 kb"}, + {"fields.price.pk-btree.index.options", R"({"block-size":"64 kb"})"}})); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); +} + +TEST(PrimaryKeyIndexDefinitionsTest, RejectsMalformedJsonOptions) { + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", "not-json"}})); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); + } + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", R"({"":"v"})"}})); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); + } +} + +TEST(PrimaryKeyIndexDefinitionsTest, RejectsDuplicateColumnWithinFamily) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price,price"}})); + ASSERT_NOK_WITH_MSG(PrimaryKeyIndexDefinitions::Create(*schema), + "pk-btree.index.columns contains duplicate column 'price'."); +} + +TEST(PrimaryKeyIndexDefinitionsTest, RejectsColumnSharedAcrossFamilies) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {Options::PK_BITMAP_INDEX_COLUMNS, "price"}})); + ASSERT_NOK_WITH_MSG(PrimaryKeyIndexDefinitions::Create(*schema), + "Column 'price' can own at most one primary-key index."); +} + +TEST(PrimaryKeyIndexDefinitionsTest, ResolvesNonScalarFamiliesAndExcludesThemFromScalar) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {Options::PK_VECTOR_INDEX_COLUMNS, "emb"}, + {"fields.emb.pk-vector.index.type", "ivf-flat"}, + {Options::PK_FULL_TEXT_INDEX_COLUMNS, "status"}})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_EQ(3, definitions.Definitions().size()); + const PrimaryKeyIndexDefinition& full_text = definitions.Definitions()[1]; + ASSERT_EQ("status", full_text.Column()); + ASSERT_EQ("full-text", full_text.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::FULL_TEXT, full_text.GetFamily()); + const PrimaryKeyIndexDefinition& embedding = definitions.Definitions()[2]; + ASSERT_EQ("emb", embedding.Column()); + ASSERT_EQ(4, embedding.FieldId()); + ASSERT_EQ("ivf-flat", embedding.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::VECTOR, embedding.GetFamily()); + std::vector scalar_definitions = definitions.ScalarDefinitions(); + ASSERT_EQ(1, scalar_definitions.size()); + ASSERT_EQ("price", scalar_definitions[0].Column()); +} + +} // namespace paimon::test diff --git a/src/paimon/core/index/pk/primary_key_index_source_file.h b/src/paimon/core/index/pk/primary_key_index_source_file.h new file mode 100644 index 000000000..c1afa3de7 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_file.h @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +namespace paimon { +/// One ordered source data file covered by a source-backed primary-key index payload. +/// +/// Two source files are interchangeable only when both the file name and the row count +/// match; coverage validation relies on this strict identity. +struct PrimaryKeyIndexSourceFile { + PrimaryKeyIndexSourceFile(std::string file_name, int64_t row_count) + : file_name(std::move(file_name)), row_count(row_count) {} + + bool operator==(const PrimaryKeyIndexSourceFile& other) const { + return file_name == other.file_name && row_count == other.row_count; + } + + bool operator!=(const PrimaryKeyIndexSourceFile& other) const { + return !(*this == other); + } + + std::string file_name; + int64_t row_count; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp new file mode 100644 index 000000000..c31c34689 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/index/pk/primary_key_index_source_meta.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/io/memory_segment_output_stream.h" +#include "paimon/common/memory/memory_segment_utils.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/io/data_input_stream.h" + +namespace paimon { +namespace { +// Each serialized entry needs at least one uint16 string length and one int64 row count, +// mirroring the defensive source file count cap of the Java deserializer. +constexpr int64_t kMinBytesPerSourceFile = sizeof(uint16_t) + sizeof(int64_t); +constexpr size_t kMaxInitialSourceFileCapacity = 1024; +} // namespace + +Result PrimaryKeyIndexSourceMeta::Create( + int32_t data_level, std::vector source_files) { + if (data_level <= 0) { + return Status::Invalid("Primary-key index data level must be positive."); + } + if (source_files.empty()) { + return Status::Invalid("An index must reference source files."); + } + for (const PrimaryKeyIndexSourceFile& source_file : source_files) { + if (source_file.row_count < 0) { + return Status::Invalid(fmt::format("Source file {} has a negative row count {}.", + source_file.file_name, source_file.row_count)); + } + } + return PrimaryKeyIndexSourceMeta(data_level, std::move(source_files)); +} + +Result PrimaryKeyIndexSourceMeta::FromIndexFile( + const IndexFileMeta& index_file) { + const std::optional& global_index_meta = index_file.GetGlobalIndexMeta(); + if (global_index_meta == std::nullopt || global_index_meta.value().source_meta == nullptr) { + return Status::Invalid( + fmt::format("Index file {} has no source metadata.", index_file.FileName())); + } + const std::shared_ptr& source_meta = global_index_meta.value().source_meta; + return Deserialize(source_meta->data(), source_meta->size()); +} + +Result PrimaryKeyIndexSourceMeta::Deserialize(const char* data, + size_t length) { + if (data == nullptr) { + return Status::Invalid("Cannot deserialize index source metadata from a null buffer."); + } + if (length > static_cast(std::numeric_limits::max())) { + return Status::Invalid( + fmt::format("Index source metadata length {} exceeds the supported maximum {}.", length, + std::numeric_limits::max())); + } + + auto input_stream = std::make_shared(data, static_cast(length)); + DataInputStream input(input_stream); + PAIMON_ASSIGN_OR_RAISE(int32_t version, input.ReadValue()); + if (version != VERSION) { + return Status::Invalid(fmt::format("Unsupported index source version: {}.", version)); + } + PAIMON_ASSIGN_OR_RAISE(int32_t data_level, input.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int32_t source_file_count, input.ReadValue()); + if (source_file_count <= 0) { + return Status::Invalid("An index must reference source files."); + } + PAIMON_ASSIGN_OR_RAISE(int64_t position, input.GetPos()); + PAIMON_ASSIGN_OR_RAISE(int64_t stream_length, input.Length()); + int64_t maximum_source_file_count = (stream_length - position) / kMinBytesPerSourceFile; + if (static_cast(source_file_count) > maximum_source_file_count) { + return Status::Invalid(fmt::format( + "Failed to deserialize index source metadata: source file count {} exceeds the " + "maximum {} allowed by the remaining bytes.", + source_file_count, maximum_source_file_count)); + } + std::vector source_files; + source_files.reserve( + std::min(static_cast(source_file_count), kMaxInitialSourceFileCapacity)); + for (int32_t i = 0; i < source_file_count; i++) { + PAIMON_ASSIGN_OR_RAISE(std::string file_name, input.ReadString()); + PAIMON_ASSIGN_OR_RAISE(int64_t row_count, input.ReadValue()); + source_files.emplace_back(std::move(file_name), row_count); + } + PAIMON_ASSIGN_OR_RAISE(position, input.GetPos()); + if (position != stream_length) { + return Status::Invalid("Unexpected trailing bytes in index source metadata."); + } + return Create(data_level, std::move(source_files)); +} + +Result> PrimaryKeyIndexSourceMeta::Serialize( + const std::shared_ptr& pool) const { + if (pool == nullptr) { + return Status::Invalid("Cannot serialize index source metadata with a null memory pool."); + } + if (source_files_.size() > static_cast(std::numeric_limits::max())) { + return Status::Invalid( + fmt::format("Index source file count {} exceeds the supported maximum {}.", + source_files_.size(), std::numeric_limits::max())); + } + + int64_t serialized_size = 3 * static_cast(sizeof(int32_t)); + for (const PrimaryKeyIndexSourceFile& source_file : source_files_) { + if (source_file.file_name.size() > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("Source file name is too long for a 16-bit length: {} bytes.", + source_file.file_name.size())); + } + int64_t entry_size = + kMinBytesPerSourceFile + static_cast(source_file.file_name.size()); + if (serialized_size > std::numeric_limits::max() - entry_size) { + return Status::Invalid(fmt::format( + "Serialized index source metadata exceeds the supported maximum {} bytes.", + std::numeric_limits::max())); + } + serialized_size += entry_size; + } + + MemorySegmentOutputStream output(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); + output.WriteValue(VERSION); + output.WriteValue(data_level_); + output.WriteValue(static_cast(source_files_.size())); + for (const PrimaryKeyIndexSourceFile& source_file : source_files_) { + auto name_length = static_cast(source_file.file_name.size()); + output.WriteValue(name_length); + output.Write(source_file.file_name.data(), name_length); + output.WriteValue(source_file.row_count); + } + PAIMON_UNIQUE_PTR bytes = MemorySegmentUtils::CopyToBytes( + output.Segments(), /*offset=*/0, static_cast(serialized_size), pool.get()); + return std::shared_ptr(std::move(bytes)); +} + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.h b/src/paimon/core/index/pk/primary_key_index_source_meta.h new file mode 100644 index 000000000..dbfa485b2 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.h @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { +class IndexFileMeta; + +/// Ordered source data files covered by a source-backed primary-key index payload. +/// +/// Wire format (version 1): big-endian int32 version, big-endian int32 data level (> 0), +/// big-endian int32 source file count (> 0), then per source file a uint16 big-endian byte +/// length, unchanged file name bytes, and a big-endian int64 row count. This matches Java +/// `writeUTF` for ASCII and non-null BMP UTF-8 file names. Java modified UTF-8 support for +/// supplementary code points requires a stream-level follow-up. Trailing bytes are rejected. +class PrimaryKeyIndexSourceMeta { + public: + static constexpr int32_t VERSION = 1; + + static Result Create( + int32_t data_level, std::vector source_files); + + /// Extracts and deserializes the source metadata carried by an index file. + static Result FromIndexFile(const IndexFileMeta& index_file); + + static Result Deserialize(const char* data, size_t length); + + Result> Serialize(const std::shared_ptr& pool) const; + + int32_t DataLevel() const { + return data_level_; + } + + const std::vector& SourceFiles() const { + return source_files_; + } + + private: + PrimaryKeyIndexSourceMeta(int32_t data_level, + std::vector source_files) + : data_level_(data_level), source_files_(std::move(source_files)) {} + + int32_t data_level_; + std::vector source_files_; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp new file mode 100644 index 000000000..44b83f5d3 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/index/pk/primary_key_index_source_meta.h" + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class PrimaryKeyIndexSourceMetaTest : public ::testing::Test { + protected: + void SetUp() override { + pool_ = GetDefaultPool(); + } + + Result SerializeToString(int32_t data_level, + std::vector source_files) { + PAIMON_ASSIGN_OR_RAISE( + PrimaryKeyIndexSourceMeta meta, + PrimaryKeyIndexSourceMeta::Create(data_level, std::move(source_files))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, meta.Serialize(pool_)); + return std::string(bytes->data(), bytes->size()); + } + + std::shared_ptr pool_; +}; + +TEST_F(PrimaryKeyIndexSourceMetaTest, SerializeMatchesGoldenBytes) { + std::vector files; + files.emplace_back("a.parquet", 100); + files.emplace_back("b.parquet", 200); + ASSERT_OK_AND_ASSIGN(std::string serialized, SerializeToString(3, files)); + + const uint8_t kExpected[] = { + 0x00, 0x00, 0x00, 0x01, // version 1 + 0x00, 0x00, 0x00, 0x03, // data level 3 + 0x00, 0x00, 0x00, 0x02, // source file count 2 + 0x00, 0x09, // writeUTF byte length of "a.parquet" + 'a', '.', 'p', 'a', 'r', 'q', 'u', 'e', 't', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, // row count 100 + 0x00, 0x09, // writeUTF byte length of "b.parquet" + 'b', '.', 'p', 'a', 'r', 'q', 'u', 'e', 't', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC8, // row count 200 + }; + std::string expected(reinterpret_cast(kExpected), sizeof(kExpected)); + ASSERT_EQ(expected, serialized); + + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta meta, PrimaryKeyIndexSourceMeta::Deserialize( + serialized.data(), serialized.size())); + ASSERT_EQ(3, meta.DataLevel()); + ASSERT_EQ(files, meta.SourceFiles()); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, RoundTripWithBmpNameAndLargeRowCount) { + std::vector files; + // Java modified UTF-8 and standard UTF-8 use identical bytes for non-null BMP text. + // Row count above 2^32 exercises the full big-endian int64 encoding. + files.emplace_back("文件-0.parquet", (int64_t{1} << 40) + 7); + files.emplace_back("data-1.parquet", 42); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta meta, + PrimaryKeyIndexSourceMeta::Create(5, files)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, meta.Serialize(pool_)); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta decoded, + PrimaryKeyIndexSourceMeta::Deserialize(bytes->data(), bytes->size())); + ASSERT_EQ(5, decoded.DataLevel()); + ASSERT_EQ(files, decoded.SourceFiles()); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, DeserializeRejectsBadHeaders) { + std::vector files; + files.emplace_back("a.parquet", 100); + ASSERT_OK_AND_ASSIGN(std::string valid, SerializeToString(3, files)); + // Layout: [0,4) version, [4,8) data level, [8,12) count, [12,14) name length, + // [14,23) name bytes, [23,31) row count. + ASSERT_EQ(static_cast(31), valid.size()); + + // Unsupported versions. + std::string version_two = valid; + version_two[3] = '\x02'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(version_two.data(), version_two.size())); + std::string version_zero = valid; + version_zero[3] = '\x00'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(version_zero.data(), version_zero.size())); + + // Source file count must be positive. + std::string zero_count = valid; + zero_count[11] = '\x00'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(zero_count.data(), zero_count.size())); + std::string negative_count = valid; + for (size_t i = 8; i < 12; i++) { + negative_count[i] = '\xFF'; + } + ASSERT_NOK( + PrimaryKeyIndexSourceMeta::Deserialize(negative_count.data(), negative_count.size())); + + // Claimed count 1000 exceeds the defensive cap allowed by the 19 remaining bytes. + std::string huge_count = valid; + huge_count[10] = '\x03'; + huge_count[11] = '\xE8'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(huge_count.data(), huge_count.size())); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, DeserializeRejectsBadPayloads) { + std::vector files; + files.emplace_back("a.parquet", 100); + ASSERT_OK_AND_ASSIGN(std::string valid, SerializeToString(3, files)); + ASSERT_EQ(static_cast(31), valid.size()); + + // Trailing bytes after a valid payload. + std::string trailing = valid + '\x00'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(trailing.data(), trailing.size())); + + // Buffer cut in the middle of the file name: only 8 of the 9 name bytes remain. + std::string cut_name = valid.substr(0, 22); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(cut_name.data(), cut_name.size())); + + // Buffer cut in the middle of the row count: only 4 of the 8 bytes remain. + std::string cut_row_count = valid.substr(0, 27); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(cut_row_count.data(), cut_row_count.size())); + + std::string negative_row_count = valid; + negative_row_count[23] = '\xFF'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(negative_row_count.data(), + negative_row_count.size())); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, CreateRejectsInvalidArguments) { + std::vector files; + files.emplace_back("a.parquet", 100); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(0, files)); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(-1, files)); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(3, {})); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(3, {{"a.parquet", -1}})); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, SerializeValidatesStringLengthAndMemoryPool) { + std::string maximum_name(std::numeric_limits::max(), 'a'); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta maximum_meta, + PrimaryKeyIndexSourceMeta::Create(1, {{maximum_name, 1}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr maximum_bytes, maximum_meta.Serialize(pool_)); + ASSERT_OK_AND_ASSIGN( + PrimaryKeyIndexSourceMeta maximum_decoded, + PrimaryKeyIndexSourceMeta::Deserialize(maximum_bytes->data(), maximum_bytes->size())); + ASSERT_EQ(maximum_meta.SourceFiles(), maximum_decoded.SourceFiles()); + + std::string oversized_name(static_cast(std::numeric_limits::max()) + 1, 'a'); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta oversized_meta, + PrimaryKeyIndexSourceMeta::Create(1, {{std::move(oversized_name), 1}})); + ASSERT_NOK_WITH_MSG(oversized_meta.Serialize(pool_), "too long for a 16-bit length"); + ASSERT_NOK_WITH_MSG(maximum_meta.Serialize(nullptr), "null memory pool"); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, DeserializeRejectsNullBuffer) { + ASSERT_NOK_WITH_MSG(PrimaryKeyIndexSourceMeta::Deserialize(nullptr, 0), "null buffer"); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, FromIndexFileDecodesSourceMeta) { + std::vector files; + files.emplace_back("a.parquet", 100); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta meta, + PrimaryKeyIndexSourceMeta::Create(7, files)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr source_meta, meta.Serialize(pool_)); + std::shared_ptr index_meta = std::make_shared("index-payload", pool_.get()); + GlobalIndexMeta global_index_meta(/*_row_range_start=*/0, /*_row_range_end=*/100, + /*_index_field_id=*/1, /*_extra_field_ids=*/std::nullopt, + index_meta, source_meta); + IndexFileMeta index_file("pk-btree", "index-file-0", /*file_size=*/64, /*row_count=*/100, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, + global_index_meta); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta decoded, + PrimaryKeyIndexSourceMeta::FromIndexFile(index_file)); + ASSERT_EQ(7, decoded.DataLevel()); + ASSERT_EQ(files, decoded.SourceFiles()); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, FromIndexFileRejectsMissingSourceMeta) { + // Index file without any global index metadata. + IndexFileMeta no_global_index("pk-btree", "index-file-1", /*file_size=*/64, /*row_count=*/100, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::FromIndexFile(no_global_index)); + + // Global index metadata whose source_meta is null. + std::shared_ptr index_meta = std::make_shared("index-payload", pool_.get()); + GlobalIndexMeta null_source_meta(/*_row_range_start=*/0, /*_row_range_end=*/100, + /*_index_field_id=*/1, /*_extra_field_ids=*/std::nullopt, + index_meta, /*_source_meta=*/nullptr); + IndexFileMeta no_source_meta("pk-btree", "index-file-2", /*file_size=*/64, /*row_count=*/100, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, + null_source_meta); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::FromIndexFile(no_source_meta)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/index/pk/primary_key_index_source_policy.h b/src/paimon/core/index/pk/primary_key_index_source_policy.h new file mode 100644 index 000000000..5251bf9a9 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_policy.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_source.h" + +namespace paimon { +/// Selects complete compacted data files for source-backed primary-key indexes. +/// +/// Only files produced by compaction on a positive data level are eligible: level 0 files +/// and appended files may still be rewritten or merged, so payloads built over them could +/// not maintain the exact per-level coverage contract. +class PrimaryKeyIndexSourcePolicy { + public: + PrimaryKeyIndexSourcePolicy() = delete; + ~PrimaryKeyIndexSourcePolicy() = delete; + + static bool ShouldWrite(const FileSource& file_source, int32_t level) { + return file_source == FileSource::Compact() && level > 0; + } + + static bool ShouldRead(const DataFileMeta& file) { + if (file.file_source == std::nullopt) { + return false; + } + return ShouldWrite(file.file_source.value(), file.level); + } +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp new file mode 100644 index 000000000..7d3b5a6d1 --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/index/pksorted/pk_sorted_bucket_index_state.h" + +#include +#include +#include +#include +#include + +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_policy.h" + +namespace paimon { +PkSortedBucketIndexState PkSortedBucketIndexState::FromActiveDataFiles( + int32_t field_id, const std::string& index_type, + const std::vector>& active_data_files, + const std::vector>& active_payloads) { + std::map> sources_by_level; + for (const std::shared_ptr& data_file : active_data_files) { + if (data_file != nullptr && PrimaryKeyIndexSourcePolicy::ShouldRead(*data_file)) { + sources_by_level[data_file->level].emplace_back(data_file->file_name, + data_file->row_count); + } + } + for (auto& level_sources : sources_by_level) { + std::sort( + level_sources.second.begin(), level_sources.second.end(), + [](const PrimaryKeyIndexSourceFile& left, const PrimaryKeyIndexSourceFile& right) { + return left.file_name < right.file_name; + }); + } + + // Match payloads against the expected level sources; anything that does not decode or + // does not exactly cover its level is rejected. + std::map>> payloads_by_level; + std::map> payload_metas_by_level; + std::vector> rejected; + for (const std::shared_ptr& payload : active_payloads) { + if (payload == nullptr) { + continue; + } + const std::optional& global_index_meta = payload->GetGlobalIndexMeta(); + if (payload->IndexType() != index_type || global_index_meta == std::nullopt || + global_index_meta->index_field_id != field_id) { + rejected.push_back(payload); + continue; + } + Result source_meta_result = + PrimaryKeyIndexSourceMeta::FromIndexFile(*payload); + if (!source_meta_result.ok()) { + rejected.push_back(payload); + continue; + } + PrimaryKeyIndexSourceMeta source_meta = std::move(source_meta_result).value(); + auto desired = sources_by_level.find(source_meta.DataLevel()); + if (desired == sources_by_level.end() || desired->second != source_meta.SourceFiles()) { + rejected.push_back(payload); + continue; + } + payloads_by_level[source_meta.DataLevel()].push_back(payload); + payload_metas_by_level[source_meta.DataLevel()].push_back(std::move(source_meta)); + } + + std::vector> groups; + std::set covered_levels; + for (const auto& level_payloads : payloads_by_level) { + int32_t level = level_payloads.first; + std::shared_ptr group; + if (level_payloads.second.size() == 1) { + group = PkSortedIndexGroup::Create(field_id, index_type, sources_by_level[level], + level_payloads.second[0], + payload_metas_by_level[level][0]); + } + if (group != nullptr) { + groups.push_back(std::move(group)); + covered_levels.insert(level); + } else { + rejected.insert(rejected.end(), level_payloads.second.begin(), + level_payloads.second.end()); + } + } + + std::vector covered; + std::vector uncovered; + for (const auto& level_sources : sources_by_level) { + auto& target = covered_levels.count(level_sources.first) > 0 ? covered : uncovered; + target.insert(target.end(), level_sources.second.begin(), level_sources.second.end()); + } + return PkSortedBucketIndexState(std::move(groups), std::move(covered), std::move(uncovered), + std::move(rejected)); +} + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h new file mode 100644 index 000000000..6923010da --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/core/index/pksorted/pk_sorted_index_group.h" +#include "paimon/core/io/data_file_meta.h" + +namespace paimon { +/// Immutable source-backed sorted-index state for one field and bucket. +/// +/// Derives the eligible per-level source sets from the active data files, matches the +/// active payloads against them, and keeps the exact validated groups. Payloads whose +/// source metadata cannot be decoded or does not exactly cover its level are rejected; +/// levels without a valid group stay uncovered and must be scanned normally. +class PkSortedBucketIndexState { + public: + static PkSortedBucketIndexState FromActiveDataFiles( + int32_t field_id, const std::string& index_type, + const std::vector>& active_data_files, + const std::vector>& active_payloads); + + const std::vector>& Groups() const { + return groups_; + } + + const std::vector& CoveredSourceFiles() const { + return covered_source_files_; + } + + const std::vector& UncoveredSourceFiles() const { + return uncovered_source_files_; + } + + const std::vector>& RejectedPayloads() const { + return rejected_payloads_; + } + + private: + PkSortedBucketIndexState(std::vector> groups, + std::vector covered_source_files, + std::vector uncovered_source_files, + std::vector> rejected_payloads) + : groups_(std::move(groups)), + covered_source_files_(std::move(covered_source_files)), + uncovered_source_files_(std::move(uncovered_source_files)), + rejected_payloads_(std::move(rejected_payloads)) {} + + std::vector> groups_; + std::vector covered_source_files_; + std::vector uncovered_source_files_; + std::vector> rejected_payloads_; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp new file mode 100644 index 000000000..d01f18c44 --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/index/pksorted/pk_sorted_bucket_index_state.h" + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/core/index/pk/primary_key_index_source_meta.h" +#include "paimon/core/index/pksorted/pk_sorted_index_group.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +class PkSortedBucketIndexStateTest : public ::testing::Test { + public: + std::shared_ptr MakeDataFile(const std::string& file_name, int64_t row_count, + int32_t level, + const std::optional& file_source) const { + return std::make_shared( + file_name, /*file_size=*/1024, row_count, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/1, /*schema_id=*/0, level, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, file_source, /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + } + + /// Builds a payload whose source metadata lists the given sources in the given order. + std::shared_ptr MakePayload( + int32_t field_id, const std::string& index_type, int32_t data_level, + const std::vector& sources, int64_t total_row_count, + int64_t row_range_start, int64_t row_range_end) const { + EXPECT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta source_meta, + PrimaryKeyIndexSourceMeta::Create(data_level, sources)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr source_meta_bytes, + source_meta.Serialize(pool_)); + return MakePayloadWithSourceMetaBytes(field_id, index_type, total_row_count, + row_range_start, row_range_end, source_meta_bytes); + } + + std::shared_ptr MakePayload( + int32_t field_id, const std::string& index_type, int32_t data_level, + const std::vector& sources, int64_t total_row_count) const { + return MakePayload(field_id, index_type, data_level, sources, total_row_count, + /*row_range_start=*/0, /*row_range_end=*/total_row_count - 1); + } + + std::shared_ptr MakePayloadWithSourceMetaBytes( + int32_t field_id, const std::string& index_type, int64_t total_row_count, + int64_t row_range_start, int64_t row_range_end, + const std::shared_ptr& source_meta_bytes) const { + GlobalIndexMeta global_index_meta(row_range_start, row_range_end, field_id, + /*extra_field_ids=*/std::nullopt, + /*index_meta=*/nullptr, source_meta_bytes); + return std::make_shared(index_type, /*file_name=*/"payload.index", + /*file_size=*/2048, total_row_count, + /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, global_index_meta); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(PkSortedBucketIndexStateTest, BuildsGroupWhenPayloadMatchesLevelSources) { + // Files are handed over unsorted; the expected source order is sorted by file name. + std::vector> data_files = { + MakeDataFile("b", 200, 5, FileSource::Compact()), + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::vector expected_sources = {{"a", 100}, {"b", 200}}; + std::shared_ptr payload = + MakePayload(/*field_id=*/7, "btree", /*data_level=*/5, expected_sources, + /*total_row_count=*/300); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_EQ(1, state.Groups().size()); + const std::shared_ptr& group = state.Groups()[0]; + ASSERT_EQ(5, group->DataLevel()); + ASSERT_EQ(300, group->TotalSourceRowCount()); + ASSERT_EQ(expected_sources, group->SourceFiles()); + ASSERT_EQ(payload, group->Payload()); + ASSERT_EQ(expected_sources, state.CoveredSourceFiles()); + ASSERT_TRUE(state.UncoveredSourceFiles().empty()); + ASSERT_TRUE(state.RejectedPayloads().empty()); +} + +TEST_F(PkSortedBucketIndexStateTest, OnlyCompactedFilesAboveLevelZeroAreSources) { + std::vector> data_files = { + MakeDataFile("level0", 10, 0, FileSource::Compact()), + MakeDataFile("appended", 20, 5, FileSource::Append()), + MakeDataFile("unknown_source", 30, 5, std::nullopt), + MakeDataFile("c", 40, 5, FileSource::Compact())}; + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_TRUE(state.CoveredSourceFiles().empty()); + std::vector expected_uncovered = {{"c", 40}}; + ASSERT_EQ(expected_uncovered, state.UncoveredSourceFiles()); + ASSERT_TRUE(state.RejectedPayloads().empty()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithMisorderedSources) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::shared_ptr payload = + MakePayload(7, "btree", 5, {{"b", 200}, {"a", 100}}, 300); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(payload, state.RejectedPayloads()[0]); + ASSERT_TRUE(state.CoveredSourceFiles().empty()); + std::vector expected_uncovered = {{"a", 100}, {"b", 200}}; + ASSERT_EQ(expected_uncovered, state.UncoveredSourceFiles()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithMismatchedSourceRowCount) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::shared_ptr payload = + MakePayload(7, "btree", 5, {{"a", 100}, {"b", 201}}, 301); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(2, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadsCoveringWrongSourceSet) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::shared_ptr missing_source_payload = + MakePayload(7, "btree", 5, {{"a", 100}}, 100); + std::shared_ptr extra_source_payload = + MakePayload(7, "btree", 5, {{"a", 100}, {"b", 200}, {"c", 50}}, 350); + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + 7, "btree", data_files, {missing_source_payload, extra_source_payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(2, state.RejectedPayloads().size()); + ASSERT_TRUE(state.CoveredSourceFiles().empty()); + ASSERT_EQ(2, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsBothPayloadsWhenLevelHasTwoCandidates) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::vector sources = {{"a", 100}, {"b", 200}}; + std::shared_ptr first_payload = MakePayload(7, "btree", 5, sources, 300); + std::shared_ptr second_payload = MakePayload(7, "btree", 5, sources, 300); + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + 7, "btree", data_files, {first_payload, second_payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(2, state.RejectedPayloads().size()); + ASSERT_TRUE(state.CoveredSourceFiles().empty()); + ASSERT_EQ(sources, state.UncoveredSourceFiles()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongFieldId) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::shared_ptr payload = + MakePayload(/*field_id=*/8, "btree", 5, {{"a", 100}}, 100); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(1, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongIndexType) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::shared_ptr payload = MakePayload(7, "bitmap", 5, {{"a", 100}}, 100); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(1, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, WrongCandidateDoesNotMaskValidPayload) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::shared_ptr valid_payload = MakePayload(7, "btree", 5, {{"a", 100}}, 100); + std::shared_ptr wrong_payload = + MakePayload(/*field_id=*/8, "btree", 5, {{"a", 100}}, 100); + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + 7, "btree", data_files, {valid_payload, wrong_payload}); + ASSERT_EQ(1, state.Groups().size()); + ASSERT_EQ(valid_payload, state.Groups()[0]->Payload()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(wrong_payload, state.RejectedPayloads()[0]); + ASSERT_TRUE(state.UncoveredSourceFiles().empty()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongRowRange) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::vector sources = {{"a", 100}, {"b", 200}}; + // The exclusive end row 300 violates the required inclusive range [0, 299]. + std::shared_ptr payload = MakePayload(7, "btree", 5, sources, 300, + /*row_range_start=*/0, + /*row_range_end=*/300); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(sources, state.UncoveredSourceFiles()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongRowCount) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::vector sources = {{"a", 100}, {"b", 200}}; + // The row range is valid but the payload row count 299 differs from the 300 source rows. + std::shared_ptr payload = MakePayload(7, "btree", 5, sources, + /*total_row_count=*/299, + /*row_range_start=*/0, + /*row_range_end=*/299); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(sources, state.UncoveredSourceFiles()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithCorruptSourceMeta) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + // Version 1 followed by a truncated data level. + std::shared_ptr corrupt_source_meta = + std::make_shared(std::string("\x00\x00\x00\x01\x00\x00", 6), pool_.get()); + std::shared_ptr payload = + MakePayloadWithSourceMetaBytes(7, "btree", /*total_row_count=*/300, /*row_range_start=*/0, + /*row_range_end=*/299, corrupt_source_meta); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(payload, state.RejectedPayloads()[0]); + ASSERT_EQ(2, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, KeepsValidLevelAndLeavesBrokenLevelUncovered) { + std::vector> data_files = { + MakeDataFile("c", 50, 4, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact()), + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::shared_ptr valid_payload = MakePayload(7, "btree", 4, {{"c", 50}}, 50); + std::shared_ptr broken_payload = + MakePayload(7, "btree", 5, {{"a", 100}, {"b", 999}}, 1099); + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + 7, "btree", data_files, {valid_payload, broken_payload}); + ASSERT_EQ(1, state.Groups().size()); + ASSERT_EQ(4, state.Groups()[0]->DataLevel()); + ASSERT_EQ(valid_payload, state.Groups()[0]->Payload()); + std::vector expected_covered = {{"c", 50}}; + ASSERT_EQ(expected_covered, state.CoveredSourceFiles()); + std::vector expected_uncovered = {{"a", 100}, {"b", 200}}; + ASSERT_EQ(expected_uncovered, state.UncoveredSourceFiles()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(broken_payload, state.RejectedPayloads()[0]); +} + +} // namespace paimon::test diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp new file mode 100644 index 000000000..7d3b7aada --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/index/pksorted/pk_sorted_index_file.h" + +#include +#include +#include + +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/index/pk/primary_key_index_source_meta.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" + +namespace paimon { +Result> PkSortedIndexFile::Build( + const DataField& field, const std::string& index_type, + const std::map& options, int32_t data_level, + const std::vector& source_files, + const std::shared_ptr& sorted_values, std::vector sorted_ordinals, + const std::shared_ptr& file_writer, bool is_external_path, + const std::shared_ptr& pool) { + // TODO(wangyong9999): Replace the all-in-memory sorted values and ordinals with an + // external sort buffer and feed the index writer in bounded batches. + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexSourceMeta source_meta, + PrimaryKeyIndexSourceMeta::Create(data_level, source_files)); + int64_t source_row_count = 0; + for (const PrimaryKeyIndexSourceFile& source_file : source_files) { + if (__builtin_add_overflow(source_row_count, source_file.row_count, &source_row_count)) { + return Status::Invalid("Source row count overflows in sorted index build."); + } + } + if (source_row_count <= 0) { + return Status::Invalid("A sorted index group must reference at least one source row."); + } + if (sorted_values == nullptr || sorted_values->length() != source_row_count || + static_cast(sorted_ordinals.size()) != source_row_count) { + return Status::Invalid( + fmt::format("Sorted index input row count {} does not match source row count {}.", + sorted_values == nullptr ? 0 : sorted_values->length(), source_row_count)); + } + std::vector seen_ordinals(static_cast(source_row_count), false); + for (int64_t ordinal : sorted_ordinals) { + if (ordinal < 0 || ordinal >= source_row_count) { + return Status::Invalid( + fmt::format("Row id {} is outside sorted index group row range [0, {}).", ordinal, + source_row_count)); + } + if (seen_ordinals[ordinal]) { + return Status::Invalid(fmt::format("Row id {} appears more than once.", ordinal)); + } + seen_ordinals[ordinal] = true; + } + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(index_type, options)); + if (indexer == nullptr) { + return Status::Invalid(fmt::format("Index type {} is not registered.", index_type)); + } + auto arrow_field = DataField::ConvertDataFieldToArrowField(field); + auto arrow_schema = arrow::schema({arrow_field}); + ArrowSchema c_arrow_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); + ScopeGuard schema_guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer, + indexer->CreateWriter(field.Name(), &c_arrow_schema, file_writer, pool)); + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr struct_array, + arrow::StructArray::Make({sorted_values}, {field.Name()})); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, &c_array)); + ScopeGuard array_guard([&]() { ArrowArrayRelease(&c_array); }); + PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array, std::move(sorted_ordinals))); + PAIMON_ASSIGN_OR_RAISE(std::vector io_metas, writer->Finish()); + if (io_metas.size() != 1) { + return Status::Invalid(fmt::format( + "Sorted index build must produce exactly one payload file, but produced {}.", + io_metas.size())); + } + const GlobalIndexIOMeta& io_meta = io_metas[0]; + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr source_meta_bytes, source_meta.Serialize(pool)); + std::optional external_path; + if (is_external_path) { + PAIMON_ASSIGN_OR_RAISE(Path path, PathUtil::ToPath(io_meta.file_path)); + external_path = path.ToString(); + } + return std::make_shared( + index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, source_row_count, + /*dv_ranges=*/std::nullopt, external_path, + GlobalIndexMeta(0, source_row_count - 1, field.Id(), + /*extra_field_ids=*/std::nullopt, io_meta.metadata, source_meta_bytes)); +} + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.h b/src/paimon/core/index/pksorted/pk_sorted_index_file.h new file mode 100644 index 000000000..1c5089e94 --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.h @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/global_index/io/global_index_file_writer.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { +/// Builds one source-backed primary-key index payload for an ordered set of physical data +/// files of a single data level. +/// +/// The caller provides all indexed values of the source group as one array sorted by +/// value, together with each value's zero-based ordinal in the ordered source group. The +/// builder writes exactly one payload file and returns its metadata carrying the +/// serialized `PrimaryKeyIndexSourceMeta`, so the payload can later be validated against +/// the active source set of its level. +class PkSortedIndexFile { + public: + PkSortedIndexFile() = delete; + ~PkSortedIndexFile() = delete; + + /// @param field The indexed field. + /// @param index_type The index algorithm identifier, e.g. "btree". + /// @param options The resolved index options (algorithm-prefixed keys included). + /// @param data_level The positive data level covered by the payload. + /// @param source_files The level's active source files ordered by file name. + /// @param sorted_values All indexed values of the source group sorted by value; nulls + /// may appear anywhere. + /// @param sorted_ordinals The group ordinal of each value, aligned with + /// `sorted_values`; every ordinal in `[0, total source rows)` must appear + /// exactly once. + /// @param file_writer The index-directory file writer of the payload's bucket. + /// @param is_external_path Whether `file_writer` resolves to an external index path. + /// @param pool The memory pool used for metadata and index construction. + /// @return Metadata for the single payload file written by the index builder. + static Result> Build( + const DataField& field, const std::string& index_type, + const std::map& options, int32_t data_level, + const std::vector& source_files, + const std::shared_ptr& sorted_values, std::vector sorted_ordinals, + const std::shared_ptr& file_writer, bool is_external_path, + const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp new file mode 100644 index 000000000..851457bef --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/index/pksorted/pk_sorted_index_group.h" + +#include +#include + +namespace paimon { +std::shared_ptr PkSortedIndexGroup::Create( + int32_t field_id, const std::string& index_type, + const std::vector& expected_sources, + const std::shared_ptr& payload, + const PrimaryKeyIndexSourceMeta& payload_source_meta) { + if (payload == nullptr || expected_sources.empty()) { + return nullptr; + } + int64_t source_row_count = 0; + std::set source_names; + for (const PrimaryKeyIndexSourceFile& source_file : expected_sources) { + if (!source_names.insert(source_file.file_name).second) { + return nullptr; + } + if (__builtin_add_overflow(source_row_count, source_file.row_count, &source_row_count)) { + return nullptr; + } + } + + const std::optional& meta = payload->GetGlobalIndexMeta(); + if (payload_source_meta.SourceFiles() != expected_sources || + index_type != payload->IndexType() || meta == std::nullopt || + meta.value().index_field_id != field_id || meta.value().row_range_start != 0 || + meta.value().row_range_end != source_row_count - 1 || + payload->RowCount() != source_row_count) { + return nullptr; + } + return std::shared_ptr(new PkSortedIndexGroup( + payload_source_meta.DataLevel(), expected_sources, payload, source_row_count)); +} + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_group.h b/src/paimon/core/index/pksorted/pk_sorted_index_group.h new file mode 100644 index 000000000..9680735dc --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_index_group.h @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/core/index/pk/primary_key_index_source_meta.h" + +namespace paimon { +/// The single validated payload group which indexes one complete data level. +/// +/// A group is only created when the payload provably covers the current active source set +/// of its data level: exactly one payload, unique source names, source order / file names / +/// row counts identical to the expected level sources, matching index type and field id, +/// a row range of exactly `[0, total source rows - 1]` and a payload row count equal to the +/// source row count sum. Anything else must be treated as uncovered. +class PkSortedIndexGroup { + public: + /// Validates one payload against the expected level sources; returns null when any + /// coverage condition fails. + static std::shared_ptr Create( + int32_t field_id, const std::string& index_type, + const std::vector& expected_sources, + const std::shared_ptr& payload, + const PrimaryKeyIndexSourceMeta& payload_source_meta); + + int32_t DataLevel() const { + return data_level_; + } + + const std::vector& SourceFiles() const { + return source_files_; + } + + const std::shared_ptr& Payload() const { + return payload_; + } + + int64_t TotalSourceRowCount() const { + return total_source_row_count_; + } + + private: + PkSortedIndexGroup(int32_t data_level, std::vector source_files, + std::shared_ptr payload, int64_t total_source_row_count) + : data_level_(data_level), + source_files_(std::move(source_files)), + payload_(std::move(payload)), + total_source_row_count_(total_source_row_count) {} + + int32_t data_level_; + std::vector source_files_; + std::shared_ptr payload_; + int64_t total_source_row_count_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index b493c80d9..b2946e83c 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "arrow/c/bridge.h" @@ -300,7 +301,8 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader auto read = std::make_unique(file_store_path_factory_, internal_read_context, pool_, compact_executor_); - return read->CreateReader(partition, bucket, files, dv_factory); + return read->CreateReader(partition, bucket, files, dv_factory, + /*local_row_ranges=*/std::nullopt); } } // namespace paimon diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index eabe84268..11fdec251 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -18,12 +18,14 @@ #include "paimon/core/operation/raw_file_split_read.h" +#include #include #include #include #include "arrow/c/abi.h" #include "arrow/c/bridge.h" +#include "fmt/format.h" #include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h" #include "paimon/common/reader/complete_row_kind_batch_reader.h" #include "paimon/common/reader/concat_batch_reader.h" @@ -32,6 +34,7 @@ #include "paimon/core/core_options.h" #include "paimon/core/deletionvectors/bitmap_deletion_vector.h" #include "paimon/core/deletionvectors/deletion_vector.h" +#include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/io/file_index_evaluator.h" #include "paimon/core/operation/internal_read_context.h" @@ -52,6 +55,30 @@ class DataFilePathFactory; class Executor; class Predicate; +namespace { + +Status ValidateFileLocalRowRanges(const std::vector>& data_files, + const std::optional>& local_row_ranges) { + if (local_row_ranges == std::nullopt) { + return Status::OK(); + } + if (data_files.size() != 1) { + return Status::Invalid("file-local row ranges require exactly one data file"); + } + const auto& file = data_files.front(); + for (const Range& range : local_row_ranges.value()) { + if (range.from < 0 || range.to < range.from || range.to >= file->row_count || + range.to >= std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("Invalid file-local row range [{}, {}] for file {} with {} rows.", + range.from, range.to, file->file_name, file->row_count)); + } + } + return Status::OK(); +} + +} // namespace + RawFileSplitRead::RawFileSplitRead(const std::shared_ptr& path_factory, const std::shared_ptr& context, const std::shared_ptr& memory_pool, @@ -64,18 +91,40 @@ RawFileSplitRead::RawFileSplitRead(const std::shared_ptr& Result> RawFileSplitRead::CreateReader( const std::shared_ptr& split) { + if (auto indexed_split = std::dynamic_pointer_cast(split)) { + PAIMON_RETURN_NOT_OK(indexed_split->Validate()); + if (!indexed_split->Scores().empty()) { + // TODO(wangyong9999): Propagate indexed scores through the primary-key + // physical-position read path. + return Status::NotImplemented( + "Primary-key reads do not support scored indexed splits yet."); + } + const std::shared_ptr& inner_split = indexed_split->GetDataSplit(); + auto inner_split_impl = std::dynamic_pointer_cast(inner_split); + if (!inner_split_impl) { + return Status::Invalid("cannot cast indexed inner split to data_split"); + } + if (inner_split_impl->DataFiles().size() != 1) { + return Status::Invalid( + "indexed splits with file-local row ranges must contain exactly one file"); + } + return CreateReader(inner_split_impl->Partition(), inner_split_impl->Bucket(), + inner_split_impl->DataFiles(), inner_split_impl->DeletionFiles(), + indexed_split->RowRanges()); + } auto data_split = std::dynamic_pointer_cast(split); if (!data_split) { return Status::Invalid("cannot cast split to data_split in RawFileSplitRead"); } return CreateReader(data_split->Partition(), data_split->Bucket(), data_split->DataFiles(), - data_split->DeletionFiles()); + data_split->DeletionFiles(), /*local_row_ranges=*/std::nullopt); } Result> RawFileSplitRead::CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& data_files, - DeletionVector::Factory dv_factory) { + DeletionVector::Factory dv_factory, const std::optional>& local_row_ranges) { + PAIMON_RETURN_NOT_OK(ValidateFileLocalRowRanges(data_files, local_row_ranges)); const auto& predicate = context_->GetPredicate(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, path_factory_->CreateDataFilePathFactory(partition, bucket)); @@ -83,7 +132,7 @@ Result> RawFileSplitRead::CreateReader( PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, raw_read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory, + local_row_ranges, data_file_path_factory, /*extra_format_options=*/{})); auto raw_readers = @@ -97,16 +146,24 @@ Result> RawFileSplitRead::CreateReader( Result> RawFileSplitRead::CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& data_files, - const std::vector>& deletion_files) { + const std::vector>& deletion_files, + const std::optional>& local_row_ranges) { auto dv_factory = DeletionVector::CreateFactory( options_.GetFileSystem(), DeletionVector::CreateDeletionFileMap(data_files, deletion_files), pool_); - return CreateReader(partition, bucket, data_files, dv_factory); + return CreateReader(partition, bucket, data_files, dv_factory, local_row_ranges); } Result RawFileSplitRead::Match(const std::shared_ptr& split, bool force_keep_delete) const { - auto split_impl = dynamic_cast(split.get()); + bool is_indexed = false; + std::shared_ptr data_split = split; + if (auto indexed_split = std::dynamic_pointer_cast(split)) { + PAIMON_RETURN_NOT_OK(indexed_split->Validate()); + data_split = indexed_split->GetDataSplit(); + is_indexed = true; + } + auto split_impl = dynamic_cast(data_split.get()); if (split_impl == nullptr) { return Status::Invalid("unexpected error, split cast to impl failed"); } @@ -114,14 +171,16 @@ Result RawFileSplitRead::Match(const std::shared_ptr& split, // for append table, always return true return true; } - bool matched = !force_keep_delete && !split_impl->IsStreaming() && split_impl->RawConvertible(); + bool matched = !force_keep_delete && !split_impl->IsStreaming() && + (is_indexed || split_impl->RawConvertible()); if (matched) { // for legacy version, we are not sure if there are delete rows, but in order to be // compatible with the query acceleration of the OLAP engine, we have generated raw // files. // Here, for the sake of correctness, we still need to perform drop delete filtering. for (const auto& file : split_impl->DataFiles()) { - if (file->delete_row_count == std::nullopt) { + if (file == nullptr || file->delete_row_count == std::nullopt || + file->delete_row_count.value() != 0) { return false; } } @@ -152,6 +211,22 @@ Result> RawFileSplitRead::ApplyIndexAndDvReader PAIMON_ASSIGN_OR_RAISE(selection, bitmap_file_index->GetBitmap()); } + // narrow the selection to the file-local row positions of an indexed split + std::optional ranges_selection; + if (ranges != std::nullopt) { + RoaringBitmap32 ranges_bitmap; + for (const Range& range : ranges.value()) { + ranges_bitmap.AddRange(static_cast(range.from), + static_cast(range.to + 1)); + } + if (selection != nullptr) { + ranges_selection = RoaringBitmap32::And(*selection, ranges_bitmap); + } else { + ranges_selection = std::move(ranges_bitmap); + } + selection = &ranges_selection.value(); + } + // prepare deletion bitmap for deletion vector std::shared_ptr deletion_vector; if (dv_factory) { diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 1580cd848..ac211b257 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -19,8 +19,7 @@ #pragma once #include -#include -#include +#include #include #include "paimon/core/core_options.h" @@ -65,16 +64,22 @@ class RawFileSplitRead : public AbstractSplitRead { const std::shared_ptr& memory_pool, const std::shared_ptr& executor); + /// Also accepts an `IndexedSplit` over a single-file data split, in which case its row + /// ranges narrow the read to the given file-local physical positions. Result> CreateReader(const std::shared_ptr& split) override; + + /// Reads with an optional selection of file-local row positions. A range selection + /// requires exactly one data file. Result> CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& files, - const std::vector>& deletion_files); + const std::vector>& deletion_files, + const std::optional>& local_row_ranges); Result> CreateReader( const BinaryRow& partition, int32_t bucket, - const std::vector>& files, - DeletionVector::Factory dv_factory); + const std::vector>& files, DeletionVector::Factory dv_factory, + const std::optional>& local_row_ranges); Result Match(const std::shared_ptr& split, bool force_keep_delete) const override; diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 569f8dfd4..6f7ae9781 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -30,6 +30,7 @@ #include "paimon/common/reader/concat_batch_reader.h" #include "paimon/common/types/data_field.h" #include "paimon/core/core_options.h" +#include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/operation/internal_read_context.h" @@ -441,8 +442,9 @@ TEST_F(RawFileSplitReadTest, TestMatch) { CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique( /*path_factory=*/nullptr, std::move(internal_context), pool_, executor); - auto create_data_split = [this](bool is_streaming, - bool raw_convertible) -> std::shared_ptr { + auto create_data_split = [this](bool is_streaming, bool raw_convertible, + std::optional delete_row_count = + 0) -> std::shared_ptr { auto meta = std::make_shared( "data-d7725088-6bd4-4e70-9ce6-714ae93b47cc-0.orc", /*file_size=*/863, /*row_count=*/1, /*min_key=*/BinaryRowGenerator::GenerateRow({std::string("Alice"), 1}, pool_.get()), @@ -456,8 +458,8 @@ TEST_F(RawFileSplitReadTest, TestMatch) { pool_.get()), /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), - /*creation_time=*/Timestamp(1743525392885ll, 0), - /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*creation_time=*/Timestamp(1743525392885ll, 0), delete_row_count, + /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); @@ -506,8 +508,49 @@ TEST_F(RawFileSplitReadTest, TestMatch) { ASSERT_FALSE(match_result); } { - ASSERT_NOK(split_read->Match(nullptr, /*force_keep_delete=*/false)); + auto data_split = std::dynamic_pointer_cast( + create_data_split(/*is_streaming=*/false, /*raw_convertible=*/false)); + auto indexed_split = std::make_shared( + data_split, std::vector{Range(0, 0)}, std::vector{0.5F}); + ASSERT_NOK_WITH_MSG(split_read->CreateReader(indexed_split), + "Primary-key reads do not support scored indexed splits yet"); + } + { + auto data_split = std::dynamic_pointer_cast( + create_data_split(/*is_streaming=*/false, /*raw_convertible=*/false)); + auto indexed_split = + std::make_shared(data_split, std::vector{Range(0, 1)}); + ASSERT_NOK_WITH_MSG(split_read->CreateReader(indexed_split), + "Invalid file-local row range [0, 1]"); + } + { + auto data_split = std::dynamic_pointer_cast(create_data_split( + /*is_streaming=*/false, /*raw_convertible=*/false, /*delete_row_count=*/std::nullopt)); + auto indexed_split = + std::make_shared(data_split, std::vector{Range(0, 0)}); + ASSERT_OK_AND_ASSIGN(bool match_result, + split_read->Match(indexed_split, /*force_keep_delete=*/false)); + ASSERT_FALSE(match_result); + } + { + auto data_split = std::dynamic_pointer_cast(create_data_split( + /*is_streaming=*/false, /*raw_convertible=*/false, /*delete_row_count=*/1)); + auto indexed_split = + std::make_shared(data_split, std::vector{Range(0, 0)}); + ASSERT_OK_AND_ASSIGN(bool match_result, + split_read->Match(indexed_split, /*force_keep_delete=*/false)); + ASSERT_FALSE(match_result); + } + { + auto data_split = std::dynamic_pointer_cast(create_data_split( + /*is_streaming=*/false, /*raw_convertible=*/false, /*delete_row_count=*/0)); + auto indexed_split = + std::make_shared(data_split, std::vector{Range(0, 0)}); + ASSERT_OK_AND_ASSIGN(bool match_result, + split_read->Match(indexed_split, /*force_keep_delete=*/false)); + ASSERT_TRUE(match_result); } + ASSERT_NOK(split_read->Match(nullptr, /*force_keep_delete=*/false)); } } // namespace paimon::test diff --git a/src/paimon/core/table/source/fallback_data_split_test.cpp b/src/paimon/core/table/source/fallback_data_split_test.cpp index 18fed0cf7..80d290560 100644 --- a/src/paimon/core/table/source/fallback_data_split_test.cpp +++ b/src/paimon/core/table/source/fallback_data_split_test.cpp @@ -27,10 +27,12 @@ #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" +#include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/fallback_table_read.h" #include "paimon/data/timestamp.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" @@ -41,6 +43,41 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { +class TrackingTableRead : public TableRead { + public: + explicit TrackingTableRead(const std::shared_ptr& pool) : TableRead(pool) {} + + Result> CreateReader( + const std::shared_ptr& split) override { + last_split_ = split; + return std::unique_ptr(); + } + + std::shared_ptr last_split_; +}; +} // namespace + +TEST(FallbackTableReadTest, RoutesIndexedSplitToMainTable) { + std::shared_ptr pool = GetDefaultPool(); + auto main_table = std::make_unique(pool); + auto fallback_table = std::make_unique(pool); + TrackingTableRead* main_table_ptr = main_table.get(); + TrackingTableRead* fallback_table_ptr = fallback_table.get(); + FallbackTableRead table_read(std::move(main_table), std::move(fallback_table), pool); + + DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/"", + /*data_files=*/{}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr data_split, + builder.IsStreaming(false).RawConvertible(true).Build()); + std::shared_ptr indexed_split = + std::make_shared(data_split, std::vector()); + + ASSERT_OK(table_read.CreateReader(indexed_split)); + ASSERT_EQ(indexed_split, main_table_ptr->last_split_); + ASSERT_EQ(nullptr, fallback_table_ptr->last_split_); +} + TEST(FallbackDataSplitTest, TestDeserialize) { std::string file_name = paimon::test::GetDataDir() + "/parquet/append_table_with_append_pt_branch.db/" diff --git a/src/paimon/core/table/source/fallback_table_read.cpp b/src/paimon/core/table/source/fallback_table_read.cpp index 6ca44215e..5f5957609 100644 --- a/src/paimon/core/table/source/fallback_table_read.cpp +++ b/src/paimon/core/table/source/fallback_table_read.cpp @@ -21,6 +21,7 @@ #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/global_index/indexed_split.h" #include "paimon/status.h" #include "paimon/table/source/data_split.h" @@ -35,6 +36,9 @@ Result> FallbackTableRead::CreateReader( return main_table_->CreateReader(fallback_data_split->GetSplit()); } } + if (std::dynamic_pointer_cast(split) != nullptr) { + return main_table_->CreateReader(split); + } auto data_split = std::dynamic_pointer_cast(split); if (!data_split) { return Status::Invalid("cannot cast split to data split"); diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp index 23b902260..208807493 100644 --- a/src/paimon/core/table/source/key_value_table_read.cpp +++ b/src/paimon/core/table/source/key_value_table_read.cpp @@ -21,6 +21,7 @@ #include +#include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/operation/merge_file_split_read.h" #include "paimon/core/operation/raw_file_split_read.h" #include "paimon/core/table/source/data_split_impl.h" @@ -74,7 +75,45 @@ void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) { Result> KeyValueTableRead::CreateReader( const std::shared_ptr& split) { - auto data_split = std::dynamic_pointer_cast(split); + std::shared_ptr dispatch_split = split; + if (auto indexed_split = std::dynamic_pointer_cast(split)) { + PAIMON_RETURN_NOT_OK(indexed_split->Validate()); + if (!indexed_split->Scores().empty()) { + // TODO(wangyong9999): Propagate indexed scores through the primary-key + // physical-position read path. + return Status::NotImplemented( + "Primary-key reads do not support scored indexed splits yet."); + } + // Primary-key indexed splits carry physical positions and are routed independently + // of the inner split's raw-convertible marker, matching Java's dedicated provider. + const std::shared_ptr& inner_split = indexed_split->GetDataSplit(); + if (!force_keep_delete_) { + bool has_raw_reader = false; + for (const auto& read : split_reads_) { + if (dynamic_cast(read.get()) != nullptr) { + has_raw_reader = true; + PAIMON_ASSIGN_OR_RAISE(bool matched, + read->Match(indexed_split, /*force_keep_delete=*/false)); + if (matched) { + return read->CreateReader(indexed_split); + } + // A manually supplied or deserialized indexed split can still reference + // legacy files. Preserve merge semantics when raw-read safety is uncertain. + dispatch_split = inner_split; + break; + } + } + if (!has_raw_reader) { + return Status::Invalid( + "create reader failed, primary-key indexed split has no raw reader."); + } + } else { + // Keeping delete rows is incompatible with physical-position pruning. Reading the + // inner split through the normal merge path preserves correctness. + dispatch_split = inner_split; + } + } + auto data_split = std::dynamic_pointer_cast(dispatch_split); if (!data_split) { return Status::Invalid("split cannot be casted to DataSplit"); } diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp new file mode 100644 index 000000000..7e3872d23 --- /dev/null +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/primary_key_index_batch_scan.h" + +#include +#include +#include +#include +#include + +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/table/source/plan_impl.h" +#include "paimon/core/table/source/primary_key_sorted_index_result.h" +#include "paimon/core/table/source/primary_key_sorted_index_scan.h" +#include "paimon/core/table/source/snapshot/snapshot_reader.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/logging.h" +#include "paimon/predicate/predicate_utils.h" + +namespace paimon { +Result> PrimaryKeyIndexBatchScan::Create( + const std::shared_ptr& snapshot_reader, + std::unique_ptr&& batch_scan, + const std::shared_ptr& table_schema, + const std::shared_ptr& path_factory, const CoreOptions& core_options, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + return std::unique_ptr(new PrimaryKeyIndexBatchScan( + snapshot_reader, std::move(batch_scan), table_schema, path_factory, core_options, pool, + definitions.ScalarDefinitions())); +} + +Result> PrimaryKeyIndexBatchScan::CreatePlan() { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_plan, batch_scan_->CreatePlan()); + if (!core_options_.GlobalIndexEnabled() || scalar_definitions_.empty() || + data_plan->SnapshotId() == std::nullopt || data_plan->Splits().empty()) { + return data_plan; + } + + std::set indexed_fields; + std::set indexed_field_ids; + for (const PrimaryKeyIndexDefinition& definition : scalar_definitions_) { + indexed_fields.insert(definition.Column()); + indexed_field_ids.insert(definition.FieldId()); + } + const std::shared_ptr& predicate = batch_scan_->GetNonPartitionPredicate(); + if (predicate == nullptr) { + return data_plan; + } + Result contains_indexed_field = + PredicateUtils::ContainAnyField(predicate, indexed_fields); + if (!contains_indexed_field.ok() || !contains_indexed_field.value()) { + return data_plan; + } + + std::vector> data_splits; + data_splits.reserve(data_plan->Splits().size()); + for (const std::shared_ptr& split : data_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (data_split == nullptr || data_split->IsStreaming()) { + return data_plan; + } + data_splits.push_back(std::move(data_split)); + } + + int64_t snapshot_id = data_plan->SnapshotId().value(); + const std::shared_ptr& snapshot_manager = + snapshot_reader_->GetSnapshotManager(); + Result snapshot_result = snapshot_manager->LoadSnapshot(snapshot_id); + if (!snapshot_result.ok()) { + static auto logger = Logger::GetLogger("PrimaryKeyIndexBatchScan"); + PAIMON_LOG_WARN(logger, + "Failed to load snapshot %ld for primary-key sorted-index planning; " + "falling back to the unindexed data plan: %s", + snapshot_id, snapshot_result.status().ToString().c_str()); + return data_plan; + } + + const std::unique_ptr& index_file_handler = + snapshot_reader_->GetIndexFileHandler(); + if (index_file_handler == nullptr) { + return data_plan; + } + std::function(const IndexManifestEntry&)> entry_filter = + [&indexed_field_ids](const IndexManifestEntry& entry) -> Result { + if (!(entry.kind == FileKind::Add()) || entry.index_file == nullptr) { + return false; + } + const std::optional& meta = entry.index_file->GetGlobalIndexMeta(); + return meta != std::nullopt && meta.value().source_meta != nullptr && + indexed_field_ids.count(meta.value().index_field_id) > 0; + }; + PAIMON_ASSIGN_OR_RAISE(std::vector index_entries, + index_file_handler->Scan(snapshot_result.value(), entry_filter)); + + PAIMON_ASSIGN_OR_RAISE(PrimaryKeySortedIndexScan::Plan index_plan, + PrimaryKeySortedIndexScan::CreatePlan( + snapshot_id, data_splits, scalar_definitions_, index_entries)); + bool has_index_group = std::any_of( + index_plan.Files().begin(), index_plan.Files().end(), + [](const PrimaryKeySortedIndexScan::FilePlan& file) { return !file.Groups().empty(); }); + if (!has_index_group) { + return data_plan; + } + PrimaryKeySortedIndexScan::ReaderFactory reader_factory = + PrimaryKeySortedIndexScan::MakeReaderFactory( + core_options_.GetFileSystem(), std::make_shared(path_factory_), + table_schema_, pool_); + PAIMON_ASSIGN_OR_RAISE( + PrimaryKeySortedIndexScan::EvaluatedPlan evaluated_plan, + PrimaryKeySortedIndexScan::Evaluate(index_plan, table_schema_, predicate, + scalar_definitions_, reader_factory)); + PAIMON_ASSIGN_OR_RAISE(std::vector> splits, + PrimaryKeySortedIndexResult::ToSplits(evaluated_plan)); + return std::make_shared(data_plan->SnapshotId(), splits); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.h b/src/paimon/core/table/source/primary_key_index_batch_scan.h new file mode 100644 index 000000000..bf5284246 --- /dev/null +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.h @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/core/index/pk/primary_key_index_definition.h" +#include "paimon/core/table/source/abstract_table_scan.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/result.h" + +namespace paimon { +/// Batch scan for primary-key tables with source-backed scalar index definitions. +/// +/// Wraps the ordinary batch scan: the data plan is computed first, then the part of the +/// scan predicate that touches indexed fields is evaluated against the validated payload +/// groups of the plan's snapshot, and covered files are narrowed to indexed splits with +/// file-local row ranges. Files without trustworthy coverage keep their normal scan; the +/// reader still applies deletion vectors and the complete original predicate. +class PrimaryKeyIndexBatchScan : public AbstractTableScan { + public: + static Result> Create( + const std::shared_ptr& snapshot_reader, + std::unique_ptr&& batch_scan, + const std::shared_ptr& table_schema, + const std::shared_ptr& path_factory, const CoreOptions& core_options, + const std::shared_ptr& pool); + + Result> CreatePlan() override; + + private: + PrimaryKeyIndexBatchScan(const std::shared_ptr& snapshot_reader, + std::unique_ptr&& batch_scan, + const std::shared_ptr& table_schema, + const std::shared_ptr& path_factory, + const CoreOptions& core_options, + const std::shared_ptr& pool, + std::vector scalar_definitions) + : AbstractTableScan(core_options, snapshot_reader), + batch_scan_(std::move(batch_scan)), + table_schema_(table_schema), + path_factory_(path_factory), + pool_(pool), + scalar_definitions_(std::move(scalar_definitions)) {} + + std::unique_ptr batch_scan_; + std::shared_ptr table_schema_; + std::shared_ptr path_factory_; + std::shared_ptr pool_; + std::vector scalar_definitions_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp new file mode 100644 index 000000000..557cd4d4e --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/primary_key_sorted_index_result.h" + +#include +#include +#include +#include + +#include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/table/source/deletion_file.h" + +namespace paimon { +namespace { +struct RangeConversion { + bool use_index; + std::vector ranges; +}; + +Result> ToSingleFileSplit( + const PrimaryKeySortedIndexScan::FilePlan& file) { + const std::shared_ptr& source = file.SourceSplit(); + std::vector> data_files{file.DataFile()}; + DataSplitImpl::Builder builder(source->Partition(), source->Bucket(), source->BucketPath(), + std::move(data_files)); + builder.WithSnapshot(source->SnapshotId()) + .WithTotalBuckets(source->TotalBuckets()) + .IsStreaming(false) + .RawConvertible(false); + if (!source->DeletionFiles().empty()) { + builder.WithDataDeletionFiles({source->DeletionFiles()[file.FileIndex()]}); + } + return builder.Build(); +} + +/// Converts sorted file-local positions to merged ranges. Sets `use_index` to false when a +/// position is invalid or the result is over-fragmented, in which case the file must fall back +/// to a normal scan. +Result ToRanges(const GlobalIndexResult& result, int64_t row_count) { + std::vector ranges; + int64_t from = -1; + int64_t to = -1; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + result.CreateIterator()); + while (iterator->HasNext()) { + int64_t position = iterator->Next(); + if (position < 0 || position >= row_count || + position >= std::numeric_limits::max()) { + return RangeConversion{/*use_index=*/false, {}}; + } + if (from < 0) { + from = position; + } else if (position != to + 1) { + if (ranges.size() >= + static_cast(PrimaryKeySortedIndexResult::kMaxIndexedRangesPerFile)) { + return RangeConversion{/*use_index=*/false, {}}; + } + ranges.emplace_back(from, to); + from = position; + } + to = position; + } + if (ranges.size() >= + static_cast(PrimaryKeySortedIndexResult::kMaxIndexedRangesPerFile)) { + return RangeConversion{/*use_index=*/false, {}}; + } + ranges.emplace_back(from, to); + return RangeConversion{/*use_index=*/true, std::move(ranges)}; +} +} // namespace + +Result>> PrimaryKeySortedIndexResult::ToSplits( + const PrimaryKeySortedIndexScan::EvaluatedPlan& evaluated_plan) { + std::map preserve_raw_splits; + for (const PrimaryKeySortedIndexScan::EvaluatedFile& evaluated_file : evaluated_plan.Files()) { + const std::shared_ptr& source_split = evaluated_file.File().SourceSplit(); + if (!source_split->RawConvertible()) { + continue; + } + auto iter = preserve_raw_splits.emplace(source_split.get(), true).first; + if (evaluated_file.IndexResult() != nullptr) { + iter->second = false; + } + } + + std::vector> splits; + std::set preserved_splits; + for (const PrimaryKeySortedIndexScan::EvaluatedFile& evaluated_file : evaluated_plan.Files()) { + const PrimaryKeySortedIndexScan::FilePlan& file = evaluated_file.File(); + const std::shared_ptr& source_split = file.SourceSplit(); + if (!source_split->RawConvertible() || preserve_raw_splits[source_split.get()]) { + // Preserve the planner's bin packing when the split cannot be read file by file + // or no file in the split has a usable index result. + if (preserved_splits.insert(source_split.get()).second) { + splits.push_back(source_split); + } + continue; + } + + const std::shared_ptr& result = evaluated_file.IndexResult(); + if (result == nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_split, + ToSingleFileSplit(file)); + splits.push_back(std::move(fallback_split)); + continue; + } + + PAIMON_ASSIGN_OR_RAISE(bool is_empty, result->IsEmpty()); + if (is_empty) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(RangeConversion range_conversion, + ToRanges(*result, file.DataFile()->row_count)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr single_file_split, + ToSingleFileSplit(file)); + if (!range_conversion.use_index) { + // The index returned an invalid or over-fragmented row position set; fall back + // to a normal scan for this file. + splits.push_back(std::move(single_file_split)); + } else { + splits.push_back(std::make_shared(std::move(single_file_split), + std::move(range_conversion.ranges), + std::vector())); + } + } + return splits; +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_result.h b/src/paimon/core/table/source/primary_key_sorted_index_result.h new file mode 100644 index 000000000..8de494ad8 --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_result.h @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/core/table/source/primary_key_sorted_index_scan.h" +#include "paimon/table/source/split.h" + +namespace paimon { +/// Converts an evaluated primary-key sorted-index plan into scan splits addressed by +/// physical data-file row positions. +/// +/// Files whose index result is missing or untrustworthy keep a normal single-file scan, +/// files with an empty result are omitted, and files with a valid result become indexed +/// splits carrying their file-local row ranges next to the aligned deletion file. Source +/// splits that are not raw-convertible are preserved unchanged. +class PrimaryKeySortedIndexResult { + public: + /// The fragmentation guard of the Java implementation: a file whose index result needs + /// more ranges falls back to a normal scan. + static constexpr int32_t kMaxIndexedRangesPerFile = 4096; + + PrimaryKeySortedIndexResult() = delete; + ~PrimaryKeySortedIndexResult() = delete; + + static Result>> ToSplits( + const PrimaryKeySortedIndexScan::EvaluatedPlan& evaluated_plan); +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp new file mode 100644 index 000000000..07796b3f0 --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -0,0 +1,563 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/primary_key_sorted_index_scan.h" + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "fmt/format.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/global_index/global_index_evaluator_impl.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/index/pksorted/pk_sorted_bucket_index_state.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/logging.h" +#include "paimon/predicate/predicate_utils.h" + +namespace paimon { +namespace { +using BucketKey = std::pair; + +struct QueryKey { + Function::Type operation; + std::vector literals; + + bool operator==(const QueryKey& other) const { + return operation == other.operation && literals == other.literals; + } +}; + +/// Shares one group payload reader and its group-scope query results across all source +/// files of the group; localizes group ordinals to file-local physical positions using the +/// ordered source row-count prefix. +class SharedGroupReader { + public: + using UnderlyingReaderFactory = std::function>()>; + + SharedGroupReader(const std::shared_ptr& group, + UnderlyingReaderFactory reader_factory) + : group_(group), reader_factory_(std::move(reader_factory)) { + const std::vector& source_files = group->SourceFiles(); + source_offsets_.reserve(source_files.size() + 1); + source_offsets_.push_back(0); + for (const PrimaryKeyIndexSourceFile& source_file : source_files) { + source_offsets_.push_back(source_offsets_.back() + source_file.row_count); + } + } + + const std::shared_ptr& Group() const { + return group_; + } + + /// Runs one group-scope query with caching; equal queries evaluate exactly once. + Result> Query( + const QueryKey& key, + const std::function>(GlobalIndexReader*)>& + query) { + for (const auto& cached : query_cache_) { + if (cached.first == key) { + if (!cached.second.status.ok()) { + return cached.second.status; + } + return cached.second.result; + } + } + Result> result = RunQuery(query); + CachedQuery cached_query; + if (result.ok()) { + cached_query.result = result.value(); + } else { + cached_query.status = result.status(); + } + query_cache_.emplace_back(key, cached_query); + return result; + } + + /// Restricts one group-scope result to the local row positions of `source_index`. + /// Any out-of-range group ordinal fails the localization so that every covered file + /// of this group falls back to a normal scan; a poison marker would not survive the + /// AND/OR combination of results from other indexes. + Result> Localize( + const std::shared_ptr& result, size_t source_index) { + if (result == nullptr) { + return std::shared_ptr(nullptr); + } + assert(source_index + 1 < source_offsets_.size()); + auto localized = localized_cache_.find(result.get()); + if (localized == localized_cache_.end()) { + PAIMON_ASSIGN_OR_RAISE(std::vector> partitions, + PartitionBySource(result)); + localized = localized_cache_.emplace(result.get(), std::move(partitions)).first; + } + return localized->second[source_index]; + } + + private: + struct CachedQuery { + Status status; + std::shared_ptr result; + }; + + Result> RunQuery( + const std::function>(GlobalIndexReader*)>& + query) { + if (!reader_status_.ok()) { + return reader_status_; + } + if (reader_ == nullptr) { + Result> reader_result = reader_factory_(); + if (!reader_result.ok()) { + reader_status_ = reader_result.status(); + return reader_status_; + } + reader_ = reader_result.value(); + if (reader_ == nullptr) { + // The index type has no usable reader; keep normal scan semantics. + return std::shared_ptr(nullptr); + } + } + return query(reader_.get()); + } + + Result>> PartitionBySource( + const std::shared_ptr& result) { + size_t source_count = source_offsets_.size() - 1; + std::vector partitions(source_count); + int64_t total_row_count = source_offsets_.back(); + size_t source_index = 0; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + result->CreateIterator()); + while (iterator->HasNext()) { + int64_t position = iterator->Next(); + if (position < 0 || position >= total_row_count) { + return Status::Invalid(fmt::format( + "Sorted index returned group ordinal {} outside the source row range " + "[0, {}).", + position, total_row_count)); + } + while (position >= source_offsets_[source_index + 1]) { + source_index++; + } + partitions[source_index].Add(position - source_offsets_[source_index]); + } + std::vector> localized; + localized.reserve(source_count); + for (RoaringBitmap64& partition : partitions) { + auto bitmap = std::make_shared(std::move(partition)); + localized.push_back(std::make_shared( + [bitmap]() -> Result { return *bitmap; })); + } + return localized; + } + + std::shared_ptr group_; + UnderlyingReaderFactory reader_factory_; + std::vector source_offsets_; + std::vector> query_cache_; + std::unordered_map>> + localized_cache_; + std::shared_ptr reader_; + Status reader_status_; +}; + +/// Restricts merged source-group ordinals to one source file's local row positions. +class FileLocalGroupReader : public GlobalIndexReader { + public: + FileLocalGroupReader(std::shared_ptr shared_reader, size_t source_index) + : shared_reader_(std::move(shared_reader)), source_index_(source_index) {} + + Result> VisitIsNotNull() override { + return Query({Function::Type::IS_NOT_NULL, {}}, + [](GlobalIndexReader* reader) { return reader->VisitIsNotNull(); }); + } + + Result> VisitIsNull() override { + return Query({Function::Type::IS_NULL, {}}, + [](GlobalIndexReader* reader) { return reader->VisitIsNull(); }); + } + + Result> VisitEqual(const Literal& literal) override { + return Query({Function::Type::EQUAL, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitEqual(literal); }); + } + + Result> VisitNotEqual(const Literal& literal) override { + return Query({Function::Type::NOT_EQUAL, {literal}}, [&literal](GlobalIndexReader* reader) { + return reader->VisitNotEqual(literal); + }); + } + + Result> VisitLessThan(const Literal& literal) override { + return Query({Function::Type::LESS_THAN, {literal}}, [&literal](GlobalIndexReader* reader) { + return reader->VisitLessThan(literal); + }); + } + + Result> VisitLessOrEqual(const Literal& literal) override { + return Query( + {Function::Type::LESS_OR_EQUAL, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitLessOrEqual(literal); }); + } + + Result> VisitGreaterThan(const Literal& literal) override { + return Query( + {Function::Type::GREATER_THAN, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitGreaterThan(literal); }); + } + + Result> VisitGreaterOrEqual( + const Literal& literal) override { + return Query( + {Function::Type::GREATER_OR_EQUAL, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitGreaterOrEqual(literal); }); + } + + Result> VisitIn( + const std::vector& literals) override { + return Query({Function::Type::IN, literals}, + [&literals](GlobalIndexReader* reader) { return reader->VisitIn(literals); }); + } + + Result> VisitNotIn( + const std::vector& literals) override { + return Query({Function::Type::NOT_IN, literals}, [&literals](GlobalIndexReader* reader) { + return reader->VisitNotIn(literals); + }); + } + + Result> VisitStartsWith(const Literal& prefix) override { + return Query({Function::Type::STARTS_WITH, {prefix}}, [&prefix](GlobalIndexReader* reader) { + return reader->VisitStartsWith(prefix); + }); + } + + Result> VisitEndsWith(const Literal& suffix) override { + return Query({Function::Type::ENDS_WITH, {suffix}}, [&suffix](GlobalIndexReader* reader) { + return reader->VisitEndsWith(suffix); + }); + } + + Result> VisitContains(const Literal& literal) override { + return Query({Function::Type::CONTAINS, {literal}}, [&literal](GlobalIndexReader* reader) { + return reader->VisitContains(literal); + }); + } + + Result> VisitLike(const Literal& literal) override { + return Query({Function::Type::LIKE, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitLike(literal); }); + } + + Result> VisitVectorSearch( + const std::shared_ptr& vector_search) override { + return Status::Invalid("Primary-key sorted index does not support vector search."); + } + + Result> VisitFullTextSearch( + const std::shared_ptr& full_text_search) override { + return Status::Invalid("Primary-key sorted index does not support full text search."); + } + + bool IsThreadSafe() const override { + return false; + } + + std::string GetIndexType() const override { + return shared_reader_->Group()->Payload()->IndexType(); + } + + private: + Result> Query( + QueryKey key, + const std::function>(GlobalIndexReader*)>& + query) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr group_result, + shared_reader_->Query(key, query)); + return shared_reader_->Localize(group_result, source_index_); + } + + std::shared_ptr shared_reader_; + size_t source_index_; +}; + +Result FindSourceIndex(const PkSortedIndexGroup& group, const DataFileMeta& data_file) { + const std::vector& source_files = group.SourceFiles(); + for (size_t i = 0; i < source_files.size(); i++) { + if (source_files[i].file_name == data_file.file_name && + source_files[i].row_count == data_file.row_count) { + return i; + } + } + return Status::Invalid(fmt::format( + "Data file {} is not covered by its sorted-index source group.", data_file.file_name)); +} + +bool SupportsIndexedRawRead(const DataSplitImpl& split) { + return std::all_of(split.DataFiles().begin(), split.DataFiles().end(), + [](const std::shared_ptr& file) { + return file != nullptr && file->delete_row_count.has_value() && + file->delete_row_count.value() == 0; + }); +} +} // namespace + +Result PrimaryKeySortedIndexScan::CreatePlan( + int64_t snapshot_id, const std::vector>& data_splits, + const std::vector& definitions, + const std::vector& index_entries) { + std::unordered_map>> payloads_by_bucket; + for (const IndexManifestEntry& entry : index_entries) { + const std::shared_ptr& payload = entry.index_file; + if (payload == nullptr || !(entry.kind == FileKind::Add())) { + continue; + } + const std::optional& meta = payload->GetGlobalIndexMeta(); + if (meta == std::nullopt || meta.value().source_meta == nullptr) { + continue; + } + payloads_by_bucket[BucketKey(entry.partition, entry.bucket)].push_back(payload); + } + + std::vector scalar_definitions = + PrimaryKeyIndexDefinitions::ScalarDefinitions(definitions); + + std::unordered_map>> data_files_by_bucket; + for (const std::shared_ptr& split : data_splits) { + if (split == nullptr) { + return Status::Invalid("Primary-key sorted-index scan received a null data split."); + } + if (split->SnapshotId() != snapshot_id) { + return Status::Invalid( + fmt::format("Data split snapshot {} does not match sorted-index scan snapshot {}.", + split->SnapshotId(), snapshot_id)); + } + if (split->IsStreaming()) { + return Status::Invalid("Primary-key sorted-index scan requires batch splits."); + } + if (!split->DeletionFiles().empty() && + split->DeletionFiles().size() != split->DataFiles().size()) { + return Status::Invalid( + "Deletion files must align with data files in a sorted-index split."); + } + std::vector>& data_files = + data_files_by_bucket[BucketKey(split->Partition(), split->Bucket())]; + data_files.insert(data_files.end(), split->DataFiles().begin(), split->DataFiles().end()); + } + + // file name -> field id -> validated group, per bucket. + std::unordered_map< + BucketKey, std::map>>> + groups_by_bucket; + for (const auto& bucket_entry : data_files_by_bucket) { + const BucketKey& bucket = bucket_entry.first; + std::vector> bucket_payloads; + auto payloads_iter = payloads_by_bucket.find(bucket); + if (payloads_iter != payloads_by_bucket.end()) { + bucket_payloads = payloads_iter->second; + } + std::set> active_source_files; + for (const std::shared_ptr& data_file : bucket_entry.second) { + active_source_files.emplace(data_file->file_name, data_file->row_count); + } + std::map>> + groups_by_source; + for (const PrimaryKeyIndexDefinition& definition : scalar_definitions) { + std::vector> definition_payloads; + for (const std::shared_ptr& payload : bucket_payloads) { + const std::optional& meta = payload->GetGlobalIndexMeta(); + if (meta != std::nullopt && definition.IndexType() == payload->IndexType() && + definition.FieldId() == meta.value().index_field_id) { + definition_payloads.push_back(payload); + } + } + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + definition.FieldId(), definition.IndexType(), bucket_entry.second, + definition_payloads); + for (const std::shared_ptr& group : state.Groups()) { + for (const PrimaryKeyIndexSourceFile& source_file : group->SourceFiles()) { + if (active_source_files.count({source_file.file_name, source_file.row_count}) == + 0) { + continue; + } + groups_by_source[source_file.file_name][definition.FieldId()] = group; + } + } + } + groups_by_bucket[bucket] = std::move(groups_by_source); + } + + std::vector files; + for (const std::shared_ptr& split : data_splits) { + auto bucket_groups = groups_by_bucket.find(BucketKey(split->Partition(), split->Bucket())); + // Legacy metadata may omit delete_row_count even when the split generator marks a split + // raw-convertible. Keep the entire original split so that the normal read path can merge + // DELETE rows with records from the other files in the split. + const bool supports_indexed_raw_read = SupportsIndexedRawRead(*split); + for (size_t file_index = 0; file_index < split->DataFiles().size(); file_index++) { + const std::shared_ptr& data_file = split->DataFiles()[file_index]; + std::map> groups; + if (supports_indexed_raw_read && bucket_groups != groups_by_bucket.end() && + data_file != nullptr) { + auto source_groups = bucket_groups->second.find(data_file->file_name); + if (source_groups != bucket_groups->second.end()) { + groups = source_groups->second; + } + } + files.emplace_back(split, static_cast(file_index), std::move(groups)); + } + } + return Plan(snapshot_id, std::move(files)); +} + +Result PrimaryKeySortedIndexScan::Evaluate( + const Plan& plan, const std::shared_ptr& table_schema, + const std::shared_ptr& predicate, + const std::vector& definitions, + const ReaderFactory& reader_factory) { + std::map definitions_by_field; + for (const PrimaryKeyIndexDefinition& definition : + PrimaryKeyIndexDefinitions::ScalarDefinitions(definitions)) { + definitions_by_field.emplace(definition.FieldId(), definition); + } + + std::unordered_map> + shared_readers; + std::vector files; + files.reserve(plan.Files().size()); + for (const FilePlan& file : plan.Files()) { + GlobalIndexEvaluatorImpl::IndexReadersCreator create_readers = + [&file, &definitions_by_field, &shared_readers, &reader_factory]( + int32_t field_id) -> Result>> { + auto definition_iter = definitions_by_field.find(field_id); + std::shared_ptr group = file.Group(field_id); + if (definition_iter == definitions_by_field.end() || group == nullptr) { + return std::vector>(); + } + auto shared_iter = shared_readers.find(group.get()); + if (shared_iter == shared_readers.end()) { + const PrimaryKeyIndexDefinition& definition = definition_iter->second; + // The shared reader outlives this file plan, so the factory owns a copy of + // the file plan instead of referencing the loop variable. + SharedGroupReader::UnderlyingReaderFactory underlying_factory = + [file_copy = file, definition, group, + &reader_factory]() -> Result> { + return reader_factory(file_copy, definition, *group); + }; + shared_iter = shared_readers + .emplace(group.get(), std::make_shared( + group, std::move(underlying_factory))) + .first; + } + PAIMON_ASSIGN_OR_RAISE(size_t source_index, FindSourceIndex(*group, *file.DataFile())); + std::vector> readers; + readers.push_back( + std::make_shared(shared_iter->second, source_index)); + return readers; + }; + GlobalIndexEvaluatorImpl evaluator(table_schema, create_readers); + Result> result = evaluator.Evaluate(predicate); + if (result.ok()) { + files.emplace_back(file, result.value()); + } else { + // Evaluation failures degrade to a normal scan for this file only. + static auto logger = Logger::GetLogger("PrimaryKeySortedIndexScan"); + PAIMON_LOG_WARN(logger, + "Failed to evaluate primary-key sorted index for data file %s; " + "falling back to a normal scan for this file: %s", + file.DataFile()->file_name.c_str(), result.status().ToString().c_str()); + files.emplace_back(file, nullptr); + } + } + return EvaluatedPlan(plan.SnapshotId(), std::move(files)); +} + +namespace { +class FsGlobalIndexFileReader : public GlobalIndexFileReader { + public: + explicit FsGlobalIndexFileReader(std::shared_ptr file_system) + : file_system_(std::move(file_system)) {} + + Result> GetInputStream( + const std::string& file_path) const override { + return file_system_->Open(file_path); + } + + private: + std::shared_ptr file_system_; +}; +} // namespace + +PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFactory( + const std::shared_ptr& file_system, + const std::shared_ptr& path_factories, + const std::shared_ptr& table_schema, const std::shared_ptr& pool) { + auto file_reader = std::make_shared(file_system); + return [path_factories, table_schema, pool, file_reader]( + const FilePlan& file, const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + if (definition.GetFamily() != PrimaryKeyIndexDefinition::Family::BTREE) { + // Only the BTree payload reader is wired up; other families keep normal scan + // semantics until their dedicated readers are supported. + return std::shared_ptr(nullptr); + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr indexer, + GlobalIndexerFactory::Get(definition.IndexType(), definition.Options())); + if (indexer == nullptr) { + return std::shared_ptr(nullptr); + } + const std::shared_ptr& split = file.SourceSplit(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + path_factories->Get(split->Partition(), split->Bucket())); + const std::shared_ptr& payload = group.Payload(); + const std::optional& payload_meta = payload->GetGlobalIndexMeta(); + if (payload_meta == std::nullopt) { + // Group validation guarantees the metadata; degrade to a normal scan of the + // covered files if it is ever violated, like the Java reader factory. + return Status::Invalid(fmt::format( + "Sorted index payload {} has no global index metadata.", payload->FileName())); + } + std::vector io_metas; + io_metas.emplace_back(path_factory->ToPath(payload), payload->FileSize(), + payload_meta.value().index_meta); + PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(definition.FieldId())); + auto arrow_field = DataField::ConvertDataFieldToArrowField(field); + auto arrow_schema = arrow::schema({arrow_field}); + ArrowSchema c_arrow_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); + ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.h b/src/paimon/core/table/source/primary_key_sorted_index_scan.h new file mode 100644 index 000000000..6ff73bbdb --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.h @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/core/index/pk/primary_key_index_definition.h" +#include "paimon/core/index/pksorted/pk_sorted_index_group.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/fs/file_system.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/global_index_result.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/predicate.h" +#include "paimon/result.h" + +namespace paimon { +/// Plans and evaluates source-backed primary-key scalar index groups in file-local +/// row-position space. +/// +/// The scan works on one captured snapshot: data splits and index manifest entries must +/// come from the same snapshot. Every active data file is associated with the validated +/// payload group of its (bucket, field, data level); files without a valid group keep an +/// empty group map and later fall back to a normal scan. Evaluation runs the predicate +/// against the group payloads once per group and query, then localizes group ordinals to +/// per-file physical row positions using the ordered source row-count prefix. +class PrimaryKeySortedIndexScan { + public: + PrimaryKeySortedIndexScan() = delete; + ~PrimaryKeySortedIndexScan() = delete; + + /// One active data file and its complete field-local payload groups. + class FilePlan { + public: + FilePlan(std::shared_ptr source_split, int32_t file_index, + std::map> groups) + : source_split_(std::move(source_split)), + file_index_(file_index), + groups_(std::move(groups)) {} + + const std::shared_ptr& SourceSplit() const { + return source_split_; + } + + int32_t FileIndex() const { + return file_index_; + } + + const std::shared_ptr& DataFile() const { + return source_split_->DataFiles()[file_index_]; + } + + std::shared_ptr Group(int32_t field_id) const { + auto iter = groups_.find(field_id); + return iter == groups_.end() ? nullptr : iter->second; + } + + const std::map>& Groups() const { + return groups_; + } + + private: + std::shared_ptr source_split_; + int32_t file_index_; + std::map> groups_; + }; + + /// Immutable groups for all source files in one captured snapshot. + class Plan { + public: + Plan(int64_t snapshot_id, std::vector files) + : snapshot_id_(snapshot_id), files_(std::move(files)) {} + + int64_t SnapshotId() const { + return snapshot_id_; + } + + const std::vector& Files() const { + return files_; + } + + private: + int64_t snapshot_id_; + std::vector files_; + }; + + /// Optional file-local index result; a null result means that the file requires a + /// normal scan. + class EvaluatedFile { + public: + EvaluatedFile(FilePlan file, std::shared_ptr result) + : file_(std::move(file)), result_(std::move(result)) {} + + const FilePlan& File() const { + return file_; + } + + const std::shared_ptr& IndexResult() const { + return result_; + } + + private: + FilePlan file_; + std::shared_ptr result_; + }; + + /// Predicate results for all source files in one captured snapshot. + class EvaluatedPlan { + public: + EvaluatedPlan(int64_t snapshot_id, std::vector files) + : snapshot_id_(snapshot_id), files_(std::move(files)) {} + + int64_t SnapshotId() const { + return snapshot_id_; + } + + const std::vector& Files() const { + return files_; + } + + private: + int64_t snapshot_id_; + std::vector files_; + }; + + /// Creates a payload reader for one validated group. Returning a null reader marks the + /// index type as unusable so affected predicates keep their normal scan semantics. + using ReaderFactory = std::function>( + const FilePlan& file, const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group)>; + + /// Associates every active data file with the validated payload groups of its bucket. + /// + /// `index_entries` must be the ADD entries of the same snapshot carrying global index + /// metadata with source metadata. Splits must be non-streaming and their deletion + /// files, when present, must align with their data files. + static Result CreatePlan(int64_t snapshot_id, + const std::vector>& data_splits, + const std::vector& definitions, + const std::vector& index_entries); + + /// Evaluates the predicate for every planned file. Evaluation failures degrade to a + /// null per-file result instead of failing the scan. + static Result Evaluate(const Plan& plan, + const std::shared_ptr& table_schema, + const std::shared_ptr& predicate, + const std::vector& definitions, + const ReaderFactory& reader_factory); + + /// Creates the default reader factory which opens the single BTree payload of each validated + /// group through the table's index directory layout. Non-BTree families resolve to a null + /// reader and therefore keep normal scan semantics. + static ReaderFactory MakeReaderFactory( + const std::shared_ptr& file_system, + const std::shared_ptr& path_factories, + const std::shared_ptr& table_schema, const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp new file mode 100644 index 000000000..a56469656 --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -0,0 +1,688 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/primary_key_sorted_index_scan.h" + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/global_index/btree/btree_index_meta.h" +#include "paimon/common/global_index/btree/key_serializer.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/index/pksorted/pk_sorted_index_file.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/table/source/primary_key_sorted_index_result.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/global_index/io/global_index_file_writer.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { +constexpr int32_t kPriceFieldId = 1; +constexpr int64_t kSnapshotId = 7; +constexpr int64_t kFileARows = 100; +constexpr int64_t kFileBRows = 200; +constexpr int64_t kTotalRows = kFileARows + kFileBRows; + +class TestGlobalIndexFileWriter : public GlobalIndexFileWriter { + public: + TestGlobalIndexFileWriter(const std::shared_ptr& fs, const std::string& base_path) + : fs_(fs), base_path_(base_path) {} + + Result NewFileName(const std::string& prefix) const override { + return fmt::format("{}-index-{}", prefix, file_counter_++); + } + + Result> NewOutputStream( + const std::string& file_name) const override { + return fs_->Create(base_path_ + "/" + file_name, true); + } + + Result GetFileSize(const std::string& file_name) const override { + PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, + fs_->GetFileStatus(base_path_ + "/" + file_name)); + return file_status.GetLen(); + } + + std::string ToPath(const std::string& file_name) const override { + return base_path_ + "/" + file_name; + } + + private: + std::shared_ptr fs_; + std::string base_path_; + mutable int64_t file_counter_ = 0; +}; + +class TestGlobalIndexFileReader : public GlobalIndexFileReader { + public: + explicit TestGlobalIndexFileReader(const std::shared_ptr& fs) : fs_(fs) {} + + Result> GetInputStream( + const std::string& file_path) const override { + return fs_->Open(file_path); + } + + private: + std::shared_ptr fs_; +}; + +/// A reader stub whose equality result is fully controlled by the test, used to exercise +/// the untrusted-position fallbacks. +class StubGlobalIndexReader : public GlobalIndexReader { + public: + explicit StubGlobalIndexReader(RoaringBitmap64 equal_result) + : equal_result_(std::move(equal_result)) {} + + StubGlobalIndexReader(RoaringBitmap64 equal_result, std::shared_ptr equal_call_count) + : equal_result_(std::move(equal_result)), equal_call_count_(std::move(equal_call_count)) {} + + Result> VisitIsNotNull() override { + return NotEvaluable(); + } + Result> VisitIsNull() override { + return NotEvaluable(); + } + Result> VisitEqual(const Literal& literal) override { + if (equal_call_count_ != nullptr) { + (*equal_call_count_)++; + } + RoaringBitmap64 copy = equal_result_; + return std::make_shared( + [bitmap = std::move(copy)]() -> Result { return bitmap; }); + } + Result> VisitNotEqual(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitLessThan(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitLessOrEqual(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitGreaterThan(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitGreaterOrEqual( + const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitIn( + const std::vector& literals) override { + return NotEvaluable(); + } + Result> VisitNotIn( + const std::vector& literals) override { + return NotEvaluable(); + } + Result> VisitStartsWith(const Literal& prefix) override { + return NotEvaluable(); + } + Result> VisitEndsWith(const Literal& suffix) override { + return NotEvaluable(); + } + Result> VisitContains(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitLike(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitVectorSearch( + const std::shared_ptr& vector_search) override { + return Status::Invalid("not supported"); + } + Result> VisitFullTextSearch( + const std::shared_ptr& full_text_search) override { + return Status::Invalid("not supported"); + } + bool IsThreadSafe() const override { + return false; + } + std::string GetIndexType() const override { + return "btree"; + } + + private: + static Result> NotEvaluable() { + return std::shared_ptr(nullptr); + } + + RoaringBitmap64 equal_result_; + std::shared_ptr equal_call_count_; +}; +} // namespace + +class PrimaryKeySortedIndexScanTest : public ::testing::Test { + protected: + void SetUp() override { + pool_ = GetDefaultPool(); + test_dir_ = UniqueTestDirectory::Create("local"); + fs_ = test_dir_->GetFileSystem(); + base_path_ = test_dir_->Str(); + + std::vector fields = { + DataField(0, arrow::field("id", arrow::int64())), + DataField(kPriceFieldId, arrow::field("price", arrow::int64())), + DataField(2, arrow::field("status", arrow::utf8())), + }; + std::map options = {{"pk-btree.index.columns", "price"}}; + table_schema_ = std::make_shared( + /*version=*/3, /*id=*/0, fields, /*highest_field_id=*/2, + /*partition_keys=*/std::vector(), + /*primary_keys=*/std::vector{"id"}, options, + /*comment=*/std::nullopt, /*time_millis=*/0); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema_)); + definitions_ = definitions.ScalarDefinitions(); + ASSERT_EQ(definitions_.size(), 1); + } + + std::shared_ptr MakeDataFile(const std::string& name, int64_t row_count, + int32_t level, const FileSource& file_source, + std::optional delete_row_count = 0) { + return std::make_shared( + name, /*file_size=*/1024, row_count, + /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), + /*key_stats=*/SimpleStats::EmptyStats(), /*value_stats=*/SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/row_count, /*schema_id=*/0, level, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1721643142456LL, 0), delete_row_count, + /*embedded_index=*/nullptr, file_source, + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + } + + Result> BuildPayload(std::vector ordinals, + const std::string& writer_base_path, + bool is_external_path) { + std::vector source_files = {{"a.parquet", kFileARows}, + {"b.parquet", kFileBRows}}; + arrow::Int64Builder values_builder; + for (int64_t i = 0; i < kTotalRows; i++) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(values_builder.Append(2 * i)); + } + std::shared_ptr sorted_values; + PAIMON_RETURN_NOT_OK_FROM_ARROW(values_builder.Finish(&sorted_values)); + PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema_->GetField(kPriceFieldId)); + auto file_writer = std::make_shared(fs_, writer_base_path); + return PkSortedIndexFile::Build(field, "btree", definitions_[0].Options(), + /*data_level=*/5, source_files, sorted_values, + std::move(ordinals), file_writer, is_external_path, pool_); + } + + Result> BuildPayload(std::vector ordinals) { + return BuildPayload(std::move(ordinals), base_path_, /*is_external_path=*/false); + } + + /// Builds the standard payload of this fixture: sources a.parquet(100) + b.parquet(200) + /// on level 5, indexed value at group ordinal `i` is `2 * i`. + Result> BuildPayload() { + std::vector ordinals; + ordinals.reserve(kTotalRows); + for (int64_t i = 0; i < kTotalRows; i++) { + ordinals.push_back(i); + } + return BuildPayload(std::move(ordinals)); + } + + std::shared_ptr MakeSplit( + const std::vector>& files, bool raw_convertible, + const std::vector>& deletion_files = {}) { + std::vector> data_files = files; + DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, + base_path_ + "/bucket-0", std::move(data_files)); + builder.WithSnapshot(kSnapshotId).IsStreaming(false).RawConvertible(raw_convertible); + if (!deletion_files.empty()) { + builder.WithDataDeletionFiles(deletion_files); + } + EXPECT_OK_AND_ASSIGN(std::shared_ptr split, builder.Build()); + return split; + } + + std::vector MakeEntries(const std::shared_ptr& payload) { + return {IndexManifestEntry(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0, payload)}; + } + + PrimaryKeySortedIndexScan::ReaderFactory PayloadReaderFactory() { + std::shared_ptr fs = fs_; + std::string base_path = base_path_; + std::shared_ptr table_schema = table_schema_; + std::shared_ptr pool = pool_; + return [fs, base_path, table_schema, pool]( + const PrimaryKeySortedIndexScan::FilePlan& file, + const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr indexer, + GlobalIndexerFactory::Get(definition.IndexType(), definition.Options())); + if (indexer == nullptr) { + return Status::Invalid("btree indexer is not registered"); + } + const std::shared_ptr& payload = group.Payload(); + std::vector io_metas; + io_metas.emplace_back(base_path + "/" + payload->FileName(), payload->FileSize(), + payload->GetGlobalIndexMeta().value().index_meta); + PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(definition.FieldId())); + auto arrow_field = DataField::ConvertDataFieldToArrowField(field); + auto arrow_schema = arrow::schema({arrow_field}); + ArrowSchema c_arrow_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); + auto file_reader = std::make_shared(fs); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); + }; + } + + Result>> PlanEvaluateConvert( + const std::vector>& splits, + const std::vector& entries, const std::shared_ptr& predicate, + const PrimaryKeySortedIndexScan::ReaderFactory& reader_factory) { + PAIMON_ASSIGN_OR_RAISE( + PrimaryKeySortedIndexScan::Plan plan, + PrimaryKeySortedIndexScan::CreatePlan(kSnapshotId, splits, definitions_, entries)); + PAIMON_ASSIGN_OR_RAISE(PrimaryKeySortedIndexScan::EvaluatedPlan evaluated, + PrimaryKeySortedIndexScan::Evaluate(plan, table_schema_, predicate, + definitions_, reader_factory)); + return PrimaryKeySortedIndexResult::ToSplits(evaluated); + } + + std::shared_ptr PriceEqual(int64_t value) { + return PredicateBuilder::Equal(/*field_index=*/1, "price", FieldType::BIGINT, + Literal(value)); + } + + std::shared_ptr pool_; + std::shared_ptr test_dir_; + std::shared_ptr fs_; + std::string base_path_; + std::shared_ptr table_schema_; + std::vector definitions_; +}; + +TEST_F(PrimaryKeySortedIndexScanTest, EqualNarrowsToSingleFileRange) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + // Value 10 sits at group ordinal 5, i.e. row 5 of a.parquet. + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + auto indexed_split = std::dynamic_pointer_cast(splits[0]); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_FALSE(inner_split->RawConvertible()); + ASSERT_EQ(inner_split->DataFiles().size(), 1); + ASSERT_EQ(inner_split->DataFiles()[0]->file_name, "a.parquet"); + ASSERT_EQ(indexed_split->RowRanges().size(), 1); + ASSERT_EQ(indexed_split->RowRanges()[0].from, 5); + ASSERT_EQ(indexed_split->RowRanges()[0].to, 5); +} + +TEST_F(PrimaryKeySortedIndexScanTest, BuildRejectsDuplicateOrdinals) { + std::vector ordinals; + ordinals.reserve(kTotalRows); + for (int64_t i = 0; i < kTotalRows; i++) { + ordinals.push_back(i); + } + ordinals[1] = 0; + ASSERT_NOK_WITH_MSG(BuildPayload(std::move(ordinals)), "Row id 0 appears more than once"); +} + +TEST_F(PrimaryKeySortedIndexScanTest, ExternalPayloadPathIsNormalized) { + std::vector ordinals; + ordinals.reserve(kTotalRows); + for (int64_t i = 0; i < kTotalRows; i++) { + ordinals.push_back(i); + } + std::string writer_base_path = base_path_ + "//"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr payload, + BuildPayload(std::move(ordinals), writer_base_path, /*is_external_path=*/true)); + ASSERT_TRUE(payload->ExternalPath().has_value()); + ASSERT_OK_AND_ASSIGN(std::string normalized_path, + PathUtil::NormalizePath(writer_base_path + "/" + payload->FileName())); + ASSERT_EQ(normalized_path, payload->ExternalPath().value()); +} + +TEST_F(PrimaryKeySortedIndexScanTest, RangeSpansFileBoundary) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + // Values in [190, 210] sit at group ordinals 95..105: rows 95..99 of a.parquet and + // rows 0..5 of b.parquet. + std::shared_ptr lower = PredicateBuilder::GreaterOrEqual( + /*field_index=*/1, "price", FieldType::BIGINT, Literal(static_cast(190))); + std::shared_ptr upper = PredicateBuilder::LessOrEqual( + /*field_index=*/1, "price", FieldType::BIGINT, Literal(static_cast(210))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({lower, upper})); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), predicate, PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 2); + auto indexed_a = std::dynamic_pointer_cast(splits[0]); + auto indexed_b = std::dynamic_pointer_cast(splits[1]); + ASSERT_TRUE(indexed_a != nullptr); + ASSERT_TRUE(indexed_b != nullptr); + ASSERT_EQ(indexed_a->RowRanges().size(), 1); + ASSERT_EQ(indexed_a->RowRanges()[0].from, 95); + ASSERT_EQ(indexed_a->RowRanges()[0].to, 99); + ASSERT_EQ(indexed_b->RowRanges().size(), 1); + ASSERT_EQ(indexed_b->RowRanges()[0].from, 0); + ASSERT_EQ(indexed_b->RowRanges()[0].to, 5); +} + +TEST_F(PrimaryKeySortedIndexScanTest, GroupAndQueryAreSharedAcrossSourceFiles) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN(PrimaryKeySortedIndexScan::Plan plan, + PrimaryKeySortedIndexScan::CreatePlan(kSnapshotId, {split}, definitions_, + MakeEntries(payload))); + ASSERT_EQ(2, plan.Files().size()); + ASSERT_EQ(plan.Files()[0].Group(kPriceFieldId), plan.Files()[1].Group(kPriceFieldId)); + + RoaringBitmap64 positions; + positions.Add(5); + auto equal_call_count = std::make_shared(0); + PrimaryKeySortedIndexScan::ReaderFactory reader_factory = + [positions, equal_call_count]( + const PrimaryKeySortedIndexScan::FilePlan& file, + const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + return std::make_shared(positions, equal_call_count); + }; + ASSERT_OK(PrimaryKeySortedIndexScan::Evaluate(plan, table_schema_, PriceEqual(10), definitions_, + reader_factory)); + ASSERT_EQ(1, *equal_call_count); +} + +TEST_F(PrimaryKeySortedIndexScanTest, EmptyResultOmitsAllFiles) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + // All indexed values are even, so 11 matches nothing. + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(11), PayloadReaderFactory())); + ASSERT_TRUE(splits.empty()); +} + +TEST_F(PrimaryKeySortedIndexScanTest, UnindexedFieldPredicateFallsBack) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/2, "status", FieldType::STRING, Literal(FieldType::STRING, "hit", 3)); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), predicate, PayloadReaderFactory())); + ASSERT_EQ(1, splits.size()); + ASSERT_EQ(split, splits[0]); +} + +TEST_F(PrimaryKeySortedIndexScanTest, UncoveredFileFallsBackOthersNarrow) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact()), + MakeDataFile("c.parquet", 50, 0, FileSource::Append())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); + // a.parquet narrows to an indexed split, b.parquet is omitted, c.parquet has no + // coverage and keeps a normal single-file scan. + ASSERT_EQ(splits.size(), 2); + auto indexed_split = std::dynamic_pointer_cast(splits[0]); + ASSERT_TRUE(indexed_split != nullptr); + auto fallback_split = std::dynamic_pointer_cast(splits[1]); + ASSERT_TRUE(fallback_split != nullptr); + ASSERT_FALSE(fallback_split->RawConvertible()); + ASSERT_EQ(fallback_split->DataFiles().size(), 1); + ASSERT_EQ(fallback_split->DataFiles()[0]->file_name, "c.parquet"); +} + +TEST_F(PrimaryKeySortedIndexScanTest, NonRawConvertibleSplitPreserved) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/false); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + ASSERT_EQ(splits[0].get(), split.get()); +} + +TEST_F(PrimaryKeySortedIndexScanTest, UnknownDeleteCountPreservesOriginalSplit) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact(), std::nullopt), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN(PrimaryKeySortedIndexScan::Plan plan, + PrimaryKeySortedIndexScan::CreatePlan(kSnapshotId, {split}, definitions_, + MakeEntries(payload))); + ASSERT_TRUE(plan.Files()[0].Groups().empty()); + ASSERT_TRUE(plan.Files()[1].Groups().empty()); + + for (int64_t value : {10, 11}) { + SCOPED_TRACE(value == 10 ? "non-empty index result" : "empty index result"); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(value), + PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + ASSERT_EQ(splits[0], split); + } +} + +TEST_F(PrimaryKeySortedIndexScanTest, NonzeroDeleteCountPreservesOriginalSplit) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact(), 1), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + ASSERT_EQ(splits[0], split); +} + +TEST_F(PrimaryKeySortedIndexScanTest, InvalidRowRangePayloadFallsBack) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + // Rebuild the payload metadata with a row range end beyond the source rows: the group + // validation must reject it and every file keeps a normal scan. + const GlobalIndexMeta& meta = payload->GetGlobalIndexMeta().value(); + auto broken_payload = std::make_shared( + payload->IndexType(), payload->FileName(), payload->FileSize(), payload->RowCount(), + std::nullopt, std::nullopt, + GlobalIndexMeta(meta.row_range_start, meta.row_range_end + 1, meta.index_field_id, + meta.extra_field_ids, meta.index_meta, meta.source_meta)); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(broken_payload), PriceEqual(10), + PayloadReaderFactory())); + ASSERT_EQ(1, splits.size()); + ASSERT_EQ(split, splits[0]); +} + +TEST_F(PrimaryKeySortedIndexScanTest, MalformedBTreeMetadataFallsBack) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + const GlobalIndexMeta& meta = payload->GetGlobalIndexMeta().value(); + auto short_key = std::make_shared(std::string(1, '\0'), pool_.get()); + auto invalid_key_meta = std::make_shared(short_key, short_key, false); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_key, + KeySerializer::SerializeKey(Literal(static_cast(10)), + arrow::int64(), pool_.get())); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr last_key, + KeySerializer::SerializeKey(Literal(static_cast(1)), arrow::int64(), pool_.get())); + auto reversed_meta = std::make_shared(first_key, last_key, false); + auto only_first_meta = std::make_shared(first_key, /*last_key=*/nullptr, false); + auto only_last_meta = std::make_shared(/*first_key=*/nullptr, last_key, false); + auto empty_nonnull_meta = + std::make_shared(/*first_key=*/nullptr, /*last_key=*/nullptr, false); + std::vector> malformed_metadata = { + nullptr, + std::make_shared(std::string(4, '\0'), pool_.get()), + invalid_key_meta->Serialize(pool_.get()), + reversed_meta->Serialize(pool_.get()), + only_first_meta->Serialize(pool_.get()), + only_last_meta->Serialize(pool_.get()), + empty_nonnull_meta->Serialize(pool_.get())}; + for (const std::shared_ptr& index_meta : malformed_metadata) { + SCOPED_TRACE(index_meta == nullptr ? "missing metadata" + : fmt::format("metadata size {}", index_meta->size())); + auto broken_payload = std::make_shared( + payload->IndexType(), payload->FileName(), payload->FileSize(), payload->RowCount(), + std::nullopt, std::nullopt, + GlobalIndexMeta(meta.row_range_start, meta.row_range_end, meta.index_field_id, + meta.extra_field_ids, index_meta, meta.source_meta)); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(broken_payload), + PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + ASSERT_EQ(splits[0], split); + } +} + +TEST_F(PrimaryKeySortedIndexScanTest, OutOfRangePositionsFailAllCoveredFiles) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + RoaringBitmap64 poisoned; + poisoned.Add(5); + poisoned.Add(kTotalRows + 10); + PrimaryKeySortedIndexScan::ReaderFactory stub_factory = + [&poisoned](const PrimaryKeySortedIndexScan::FilePlan& file, + const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + return std::make_shared(poisoned); + }; + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), stub_factory)); + // Both covered files fall back together, preserving the planner's original bin packing. + ASSERT_EQ(1, splits.size()); + ASSERT_EQ(split, splits[0]); +} + +TEST_F(PrimaryKeySortedIndexScanTest, OverFragmentedResultFallsBack) { + // One data file, 20000 rows; every second row selected produces > 4096 ranges. + std::vector source_files = {{"big.parquet", 20000}}; + std::shared_ptr split = MakeSplit( + {MakeDataFile("big.parquet", 20000, 5, FileSource::Compact())}, /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr source_meta_bytes, ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexSourceMeta source_meta, + PrimaryKeyIndexSourceMeta::Create(5, source_files)); + return source_meta.Serialize(pool_); + }())); + auto big_payload = std::make_shared( + "btree", "big-index-file", /*file_size=*/1, /*row_count=*/20000, std::nullopt, std::nullopt, + GlobalIndexMeta(0, 19999, kPriceFieldId, std::nullopt, nullptr, source_meta_bytes)); + RoaringBitmap64 fragmented; + for (int64_t i = 0; i < 20000; i += 2) { + fragmented.Add(i); + } + PrimaryKeySortedIndexScan::ReaderFactory stub_factory = + [&fragmented]( + const PrimaryKeySortedIndexScan::FilePlan& file, + const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + return std::make_shared(fragmented); + }; + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(big_payload), PriceEqual(10), stub_factory)); + ASSERT_EQ(splits.size(), 1); + ASSERT_TRUE(std::dynamic_pointer_cast(splits[0]) == nullptr); +} + +TEST_F(PrimaryKeySortedIndexScanTest, DeletionFileStaysAlignedWithIndexedFile) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + DeletionFile deletion_file("dv-a", /*offset=*/0, /*length=*/16, /*cardinality=*/1); + std::shared_ptr split = MakeSplit( + {MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true, {std::optional(deletion_file), std::nullopt}); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + auto indexed_split = std::dynamic_pointer_cast(splits[0]); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_EQ(inner_split->DeletionFiles().size(), 1); + ASSERT_TRUE(inner_split->DeletionFiles()[0] != std::nullopt); + ASSERT_EQ(inner_split->DeletionFiles()[0].value().path, "dv-a"); +} + +TEST_F(PrimaryKeySortedIndexScanTest, SnapshotMismatchIsRejected) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::vector> files = { + MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact())}; + DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, base_path_, + std::move(files)); + builder.WithSnapshot(kSnapshotId + 1).IsStreaming(false).RawConvertible(true); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, builder.Build()); + ASSERT_NOK(PrimaryKeySortedIndexScan::CreatePlan(kSnapshotId, {split}, definitions_, + MakeEntries(payload))); +} + +} // namespace paimon::test diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader.h b/src/paimon/core/table/source/snapshot/snapshot_reader.h index 315ca6e07..c425fd863 100644 --- a/src/paimon/core/table/source/snapshot/snapshot_reader.h +++ b/src/paimon/core/table/source/snapshot/snapshot_reader.h @@ -92,6 +92,10 @@ class SnapshotReader { return scan_->GetSnapshotManager(); } + const std::unique_ptr& GetIndexFileHandler() const { + return index_file_handler_; + } + std::shared_ptr GetNonPartitionPredicate() const { return scan_->GetNonPartitionPredicate(); } diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 1a713834b..a543a051c 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -33,6 +33,7 @@ #include "paimon/common/utils/options_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" #include "paimon/core/manifest/index_manifest_file.h" #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_list.h" @@ -52,6 +53,7 @@ #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" #include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/primary_key_index_batch_scan.h" #include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/table/source/realtime_table_scan.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" @@ -342,12 +344,22 @@ Result> NewDataTableScan(const std::shared_ptrGetSnapshotManager(), core_options.GetFileSystem(), context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); } - if (!core_options.DataEvolutionEnabled()) { - return batch_scan; + if (core_options.DataEvolutionEnabled()) { + return std::make_unique( + context->GetPath(), snapshot_reader, std::move(batch_scan), + context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), + context->GetExecutor()); + } + if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + if (!definitions.ScalarDefinitions().empty()) { + return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), + table_schema, path_factory, core_options, + context->GetMemoryPool()); + } } - return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), context->GetGlobalIndexResult(), - core_options, context->GetMemoryPool(), context->GetExecutor()); + return batch_scan; } } // namespace diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt index 6c1fdee17..75147ce60 100644 --- a/test/inte/CMakeLists.txt +++ b/test/inte/CMakeLists.txt @@ -50,6 +50,13 @@ if(PAIMON_BUILD_TESTS) test_utils_static ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(primary_key_sorted_index_inte_test + STATIC_LINK_LIBS + paimon_shared + ${TEST_STATIC_LINK_LIBS} + test_utils_static + ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(write_and_read_inte_test STATIC_LINK_LIBS paimon_shared diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp new file mode 100644 index 000000000..8c86f4e04 --- /dev/null +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/test_helper.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +using Row = std::tuple; + +class TrackingLocalFileSystem : public LocalFileSystem { + public: + Result> Open(const std::string& path) const override { + { + std::scoped_lock lock(mutex_); + opened_paths_.push_back(path); + } + return LocalFileSystem::Open(path); + } + + std::set OpenedIndexPaths() const { + std::scoped_lock lock(mutex_); + std::set index_paths; + for (const std::string& path : opened_paths_) { + if (path.find("/index/index-") != std::string::npos) { + index_paths.insert(path); + } + } + return index_paths; + } + + private: + mutable std::mutex mutex_; + mutable std::vector opened_paths_; +}; + +std::vector ExpectedScoreZeroRows(int64_t snapshot_id) { + std::vector rows; + for (int32_t id = 10; id <= 2000; id += 10) { + if (snapshot_id >= 4 && (id == 10 || id == 20)) { + continue; + } + rows.emplace_back(-1, id, 0, "keep"); + } + if (snapshot_id >= 3) { + rows.emplace_back(-1, 2001, 0, "keep"); + } + return rows; +} + +std::vector ExpectedOrRowsAtSnapshot4() { + std::vector rows; + for (int32_t id = 1; id <= 2000; id++) { + if (id == 10 || id == 20 || id % 2 != 0) { + continue; + } + rows.emplace_back(-1, id, id % 10, "keep"); + } + rows.emplace_back(-1, 2001, 0, "keep"); + return rows; +} + +std::vector ExpectedResidualRowsAtSnapshot4() { + std::vector rows; + for (int32_t id = 30; id <= 1500; id += 10) { + rows.emplace_back(-1, id, 0, "keep"); + } + return rows; +} + +std::vector ExpectedPartitionRowsAtSnapshot2() { + std::vector rows; + for (int32_t pt = 1; pt <= 2; pt++) { + for (int32_t id = 1; id <= 100; id++) { + rows.emplace_back(pt, id, id % 10, id % 2 == 0 ? "keep" : "drop"); + } + } + return rows; +} + +std::vector ExpectedFallbackRows() { + std::vector rows; + for (int32_t id = 20; id <= 100; id += 10) { + rows.emplace_back(1, id, 0, "keep"); + } + rows.emplace_back(1, 101, 0, "keep"); + for (int32_t id = 10; id <= 100; id += 10) { + rows.emplace_back(2, id, 0, "keep"); + } + return rows; +} + +std::pair CountSplitKinds(const std::shared_ptr& plan) { + size_t indexed_count = 0; + size_t data_count = 0; + for (const std::shared_ptr& split : plan->Splits()) { + if (std::dynamic_pointer_cast(split) != nullptr) { + indexed_count++; + } else if (std::dynamic_pointer_cast(split) != nullptr) { + data_count++; + } + } + return {indexed_count, data_count}; +} + +} // namespace + +class PrimaryKeySortedIndexInteTest : public ::testing::Test, + public ::testing::WithParamInterface { + protected: + void SetUp() override { + format_ = GetParam(); + } + + std::string TablePath(bool partitioned) const { + const std::string table_name = partitioned ? "pk_btree_partitioned_e2e" : "pk_btree_e2e"; + return PathUtil::JoinPath(GetDataDir(), + fmt::format("{}/{}.db/{}", format_, table_name, table_name)); + } + + std::shared_ptr ScoreEqual(bool partitioned, int32_t score) const { + return PredicateBuilder::Equal(partitioned ? 2 : 1, "score", FieldType::INT, + Literal(score)); + } + + std::shared_ptr ScoreGreaterOrEqual(bool partitioned, int32_t score) const { + return PredicateBuilder::GreaterOrEqual(partitioned ? 2 : 1, "score", FieldType::INT, + Literal(score)); + } + + std::shared_ptr TagEqual(bool partitioned, const std::string& tag) const { + return PredicateBuilder::Equal(partitioned ? 3 : 2, "tag", FieldType::STRING, + Literal(FieldType::STRING, tag.data(), tag.size())); + } + + std::shared_ptr IdLessOrEqual(bool partitioned, int32_t id) const { + return PredicateBuilder::LessOrEqual(partitioned ? 1 : 0, "id", FieldType::INT, + Literal(id)); + } + + Result> Scan( + bool partitioned, const std::shared_ptr& predicate, + const std::optional& snapshot_id, bool index_enabled, + const std::vector>& partition_filters, + const std::string& branch, const std::shared_ptr& file_system) const { + ScanContextBuilder builder(TablePath(partitioned)); + builder.SetPredicate(predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, + index_enabled ? "true" : "false"); + if (snapshot_id != std::nullopt) { + builder.AddOption(Options::SCAN_SNAPSHOT_ID, fmt::format("{}", snapshot_id.value())); + } + if (!partition_filters.empty()) { + builder.SetPartitionFilter(partition_filters); + } + if (!branch.empty()) { + builder.AddOption(Options::BRANCH, branch); + } + if (file_system != nullptr) { + builder.WithFileSystem(file_system); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + TableScan::Create(std::move(context))); + return table_scan->CreatePlan(); + } + + Result> Read(bool partitioned, const std::shared_ptr& predicate, + const std::vector>& splits, + const std::map& options, + const std::shared_ptr& file_system) const { + ReadContextBuilder builder(TablePath(partitioned)); + if (partitioned) { + builder.SetReadFieldNames({"pt", "id", "score", "tag"}); + } else { + builder.SetReadFieldNames({"id", "score", "tag"}); + } + builder.SetPredicate(predicate).EnablePredicateFilter(true); + for (const auto& [key, value] : options) { + builder.AddOption(key, value); + } + if (file_system != nullptr) { + builder.WithFileSystem(file_system); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + table_read->CreateReader(splits)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + ReadResultCollector::CollectResult(batch_reader.get())); + if (result == nullptr) { + return std::vector(); + } + + auto struct_type = std::dynamic_pointer_cast(result->type()); + if (struct_type == nullptr) { + return Status::Invalid("primary-key E2E read result is not a struct array"); + } + int32_t pt_field = partitioned ? struct_type->GetFieldIndex("pt") : -1; + int32_t id_field = struct_type->GetFieldIndex("id"); + int32_t score_field = struct_type->GetFieldIndex("score"); + int32_t tag_field = struct_type->GetFieldIndex("tag"); + if (id_field < 0 || score_field < 0 || tag_field < 0 || (partitioned && pt_field < 0)) { + return Status::Invalid(fmt::format("unexpected primary-key E2E result type {}", + result->type()->ToString())); + } + + std::vector rows; + for (const std::shared_ptr& chunk : result->chunks()) { + auto struct_array = std::dynamic_pointer_cast(chunk); + if (struct_array == nullptr) { + return Status::Invalid("primary-key E2E result chunk is not a struct array"); + } + auto id_array = + std::dynamic_pointer_cast(struct_array->field(id_field)); + auto score_array = + std::dynamic_pointer_cast(struct_array->field(score_field)); + auto tag_array = + std::dynamic_pointer_cast(struct_array->field(tag_field)); + std::shared_ptr pt_array; + if (partitioned) { + pt_array = + std::dynamic_pointer_cast(struct_array->field(pt_field)); + } + if (id_array == nullptr || score_array == nullptr || tag_array == nullptr || + (partitioned && pt_array == nullptr)) { + return Status::Invalid("unexpected primary-key E2E result column type"); + } + for (int64_t row = 0; row < struct_array->length(); row++) { + if (id_array->IsNull(row) || score_array->IsNull(row) || tag_array->IsNull(row) || + (partitioned && pt_array->IsNull(row))) { + return Status::Invalid("unexpected null in primary-key E2E result"); + } + rows.emplace_back(partitioned ? pt_array->Value(row) : -1, id_array->Value(row), + score_array->Value(row), tag_array->GetString(row)); + } + } + std::sort(rows.begin(), rows.end()); + return rows; + } + + std::string format_; +}; + +TEST_P(PrimaryKeySortedIndexInteTest, HistoricalSnapshotsMixedFilesAndDisabledIndex) { + std::shared_ptr predicate = ScoreEqual(/*partitioned=*/false, 0); + const std::vector snapshot_ids = {2, 3, 5}; + for (int64_t snapshot_id : snapshot_ids) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + Scan(/*partitioned=*/false, predicate, snapshot_id, /*index_enabled=*/true, + /*partition_filters=*/{}, /*branch=*/"", /*file_system=*/nullptr)); + ASSERT_EQ(snapshot_id, plan->SnapshotId()); + const auto [indexed_count, data_count] = CountSplitKinds(plan); + ASSERT_GT(indexed_count, 0); + if (snapshot_id == 3) { + ASSERT_GT(data_count, 0); + } else { + ASSERT_EQ(0, data_count); + } + ASSERT_OK_AND_ASSIGN(std::vector rows, + Read(/*partitioned=*/false, predicate, plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedScoreZeroRows(snapshot_id), rows); + } + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr disabled_plan, + Scan(/*partitioned=*/false, predicate, /*snapshot_id=*/5, /*index_enabled=*/false, + /*partition_filters=*/{}, /*branch=*/"", /*file_system=*/nullptr)); + const auto [disabled_indexed_count, disabled_data_count] = CountSplitKinds(disabled_plan); + ASSERT_EQ(0, disabled_indexed_count); + ASSERT_GT(disabled_data_count, 0); + ASSERT_OK_AND_ASSIGN( + std::vector disabled_rows, + Read(/*partitioned=*/false, predicate, disabled_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedScoreZeroRows(/*snapshot_id=*/5), disabled_rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, MultiSourceOrdinalsBecomeBoundedFileLocalRanges) { + std::shared_ptr predicate = ScoreEqual(/*partitioned=*/false, 0); + auto tracking_file_system = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + Scan(/*partitioned=*/false, predicate, /*snapshot_id=*/2, /*index_enabled=*/true, + /*partition_filters=*/{}, /*branch=*/"", tracking_file_system)); + + std::set source_files; + int64_t selected_rows = 0; + for (const std::shared_ptr& split : plan->Splits()) { + auto indexed_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_FALSE(inner_split->RawConvertible()); + ASSERT_EQ(1, inner_split->DataFiles().size()); + const std::shared_ptr& file = inner_split->DataFiles()[0]; + source_files.insert(file->file_name); + for (const Range& range : indexed_split->RowRanges()) { + ASSERT_GE(range.from, 0); + ASSERT_GE(range.to, range.from); + ASSERT_LT(range.to, file->row_count); + selected_rows += range.to - range.from + 1; + } + } + ASSERT_GE(source_files.size(), 2); + ASSERT_EQ(200, selected_rows); + ASSERT_EQ(1, tracking_file_system->OpenedIndexPaths().size()); + + ASSERT_OK_AND_ASSIGN(std::vector rows, + Read(/*partitioned=*/false, predicate, plan->Splits(), /*options=*/{}, + tracking_file_system)); + ASSERT_EQ(ExpectedScoreZeroRows(/*snapshot_id=*/2), rows); + ASSERT_OK_AND_ASSIGN(std::vector ranges_only_rows, + Read(/*partitioned=*/false, /*predicate=*/nullptr, plan->Splits(), + /*options=*/{}, tracking_file_system)); + ASSERT_EQ(ExpectedScoreZeroRows(/*snapshot_id=*/2), ranges_only_rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, DeletionVectorResidualAndBooleanFallback) { + std::shared_ptr score = ScoreEqual(/*partitioned=*/false, 0); + std::shared_ptr id_upper_bound = IdLessOrEqual(/*partitioned=*/false, /*id=*/1500); + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_predicate, + PredicateBuilder::And({score, id_upper_bound})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_plan, + Scan(/*partitioned=*/false, and_predicate, /*snapshot_id=*/4, + /*index_enabled=*/true, /*partition_filters=*/{}, /*branch=*/"", + /*file_system=*/nullptr)); + bool has_deletion_vector = false; + for (const std::shared_ptr& split : and_plan->Splits()) { + auto indexed_split = std::dynamic_pointer_cast(split); + if (indexed_split == nullptr) { + continue; + } + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + for (const std::optional& deletion_file : inner_split->DeletionFiles()) { + has_deletion_vector = has_deletion_vector || deletion_file.has_value(); + } + } + ASSERT_TRUE(has_deletion_vector); + ASSERT_OK_AND_ASSIGN(std::vector and_rows, Read(/*partitioned=*/false, and_predicate, + and_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedResidualRowsAtSnapshot4(), and_rows); + + std::shared_ptr tag = TagEqual(/*partitioned=*/false, "keep"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_predicate, + PredicateBuilder::Or({score, tag})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_plan, + Scan(/*partitioned=*/false, or_predicate, /*snapshot_id=*/4, + /*index_enabled=*/true, /*partition_filters=*/{}, /*branch=*/"", + /*file_system=*/nullptr)); + const auto [or_indexed_count, or_data_count] = CountSplitKinds(or_plan); + ASSERT_EQ(0, or_indexed_count); + ASSERT_GT(or_data_count, 0); + ASSERT_OK_AND_ASSIGN(std::vector or_rows, Read(/*partitioned=*/false, or_predicate, + or_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedOrRowsAtSnapshot4(), or_rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, PartitionsBucketsAndIndexPaths) { + std::shared_ptr predicate = ScoreGreaterOrEqual(/*partitioned=*/true, /*score=*/0); + auto tracking_file_system = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + Scan(/*partitioned=*/true, predicate, /*snapshot_id=*/2, /*index_enabled=*/true, + /*partition_filters=*/{}, /*branch=*/"", tracking_file_system)); + + std::set partitions; + std::set buckets; + int64_t selected_rows = 0; + for (const std::shared_ptr& split : plan->Splits()) { + auto indexed_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_EQ(1, inner_split->DataFiles().size()); + partitions.insert(inner_split->Partition().GetInt(0)); + buckets.insert(inner_split->Bucket()); + const int64_t row_count = inner_split->DataFiles()[0]->row_count; + for (const Range& range : indexed_split->RowRanges()) { + ASSERT_GE(range.from, 0); + ASSERT_LT(range.to, row_count); + selected_rows += range.to - range.from + 1; + } + } + ASSERT_EQ((std::set{1, 2}), partitions); + ASSERT_EQ((std::set{0, 1}), buckets); + ASSERT_EQ(200, selected_rows); + ASSERT_FALSE(tracking_file_system->OpenedIndexPaths().empty()); + + ASSERT_OK_AND_ASSIGN(std::vector rows, + Read(/*partitioned=*/true, predicate, plan->Splits(), /*options=*/{}, + tracking_file_system)); + ASSERT_EQ(ExpectedPartitionRowsAtSnapshot2(), rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, FallbackTableReadRoutesMainAndFallbackSplits) { + std::shared_ptr predicate = ScoreEqual(/*partitioned=*/true, 0); + const std::vector> main_partition = {{{"pt", "1"}}}; + const std::vector> fallback_partition = {{{"pt", "2"}}}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr main_plan, + Scan(/*partitioned=*/true, predicate, /*snapshot_id=*/5, /*index_enabled=*/true, + main_partition, /*branch=*/"", /*file_system=*/nullptr)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, + Scan(/*partitioned=*/true, predicate, /*snapshot_id=*/std::nullopt, + /*index_enabled=*/false, fallback_partition, /*branch=*/"fallback", + /*file_system=*/nullptr)); + + std::vector> routed_splits; + for (const std::shared_ptr& split : main_plan->Splits()) { + ASSERT_TRUE(std::dynamic_pointer_cast(split) != nullptr); + routed_splits.push_back(split); + } + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + routed_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_FALSE(main_plan->Splits().empty()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + + const std::map read_options = { + {Options::SCAN_FALLBACK_BRANCH, "fallback"}}; + ASSERT_OK_AND_ASSIGN(std::vector rows, + Read(/*partitioned=*/true, predicate, routed_splits, read_options, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedFallbackRows(), rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, ScoredSplitFailsBeforeForceKeepDeleteFallback) { + std::shared_ptr predicate = ScoreEqual(/*partitioned=*/false, 0); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + Scan(/*partitioned=*/false, predicate, /*snapshot_id=*/2, /*index_enabled=*/true, + /*partition_filters=*/{}, /*branch=*/"", /*file_system=*/nullptr)); + ASSERT_FALSE(plan->Splits().empty()); + auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_FALSE(indexed_split->RowRanges().empty()); + const Range& first_range = indexed_split->RowRanges()[0]; + auto scored_split = std::make_shared( + inner_split, std::vector{Range(first_range.from, first_range.from)}, + std::vector{0.5F}); + + ReadContextBuilder builder(TablePath(/*partitioned=*/false)); + builder.SetReadFieldNames({"id", "score", "tag"}) + .SetPredicate(predicate) + .EnablePredicateFilter(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(context))); + auto* key_value_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(key_value_table_read != nullptr); + key_value_table_read->ForceKeepDelete(true); + ASSERT_NOK_WITH_MSG(key_value_table_read->CreateReader(scored_split), + "Primary-key reads do not support scored indexed splits yet"); +} + +std::vector PrimaryKeySortedIndexFormats() { + std::vector formats = {"parquet"}; +#ifdef PAIMON_ENABLE_ORC + formats.emplace_back("orc"); +#endif + return formats; +} + +INSTANTIATE_TEST_SUITE_P(FileFormat, PrimaryKeySortedIndexInteTest, + ::testing::ValuesIn(PrimaryKeySortedIndexFormats())); + +} // namespace paimon::test diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/README b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/README new file mode 100644 index 000000000..4f86a4af4 --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/README @@ -0,0 +1,24 @@ +id:int score:int tag:string +primary key: id +no partition key +bucket count: 1 + +Generated by Apache Paimon Java release-2.0.0. +file format: orc +deletion-vectors.enabled: true +deletion-vectors.merge-on-read: false +write-only: true (explicit fixture commits produce snapshots 2, 4, 5) +primary-key btree index: score +target-file-size: 8 kb +compaction.force-rewrite-all-files: true + +Snapshot semantics: +snapshot-1 APPEND: ids 1..2000; score=id%10; even tag=keep, odd tag=drop. +snapshot-2 COMPACT: full compaction builds source-backed score BTree payloads; source-file counts per payload are [2]; 2000 rows. +snapshot-3 APPEND: insert (2001,0,keep) and (2002,5,late_drop); indexed compacted files and a visible unindexed APPEND-source file coexist; 2002 rows. +snapshot-4 COMPACT: delete id=10 and update id=20 to (77,updated), then lookup compact in one fixture commit; deletion vectors coexist with indexed files; 2001 rows. +snapshot-5 COMPACT: full compaction rebuilds the BTree source groups; 2001 rows. + +Fixture construction note: with deletion vectors enabled and merge-on-read disabled, Java and C++ batch scans skip literal level-0 files. Snapshot 3 therefore promotes Java-prepared files to the maximum manifest level while retaining FileSource.APPEND. The source-backed PK BTree policy indexes only COMPACT files above level 0, making these files visible but deliberately unindexed. Snapshot 4 publishes the Java-prepared compact outputs and index changes while omitting their transient, never-active append inputs. These are test-fixture constructions, not ordinary end-user write workflows. + +At snapshots 4 and 5, predicate score=0 AND id<=1500 returns ids 30,40,...,1500; id=10 is deleted and id=20 has score 77. diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-5ee15f36-7a45-4651-8c81-a55e49dbf9ff-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-5ee15f36-7a45-4651-8c81-a55e49dbf9ff-0.orc new file mode 100644 index 000000000..e47187451 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-5ee15f36-7a45-4651-8c81-a55e49dbf9ff-0.orc differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-0.orc new file mode 100644 index 000000000..16bc68db9 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-0.orc differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-1.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-1.orc new file mode 100644 index 000000000..fa46994ff Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-1.orc differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c917f42c-c01c-4d16-9c2a-cb89c60d8d6a-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c917f42c-c01c-4d16-9c2a-cb89c60d8d6a-0.orc new file mode 100644 index 000000000..2bb754553 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c917f42c-c01c-4d16-9c2a-cb89c60d8d6a-0.orc differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-0.orc new file mode 100644 index 000000000..db298b168 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-0.orc differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-1.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-1.orc new file mode 100644 index 000000000..1f089d71f Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-1.orc differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-ed9149c4-c451-4780-a2cd-fe8a74ee2558-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-ed9149c4-c451-4780-a2cd-fe8a74ee2558-0.orc new file mode 100644 index 000000000..d2788f2c5 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-ed9149c4-c451-4780-a2cd-fe8a74ee2558-0.orc differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-0.orc new file mode 100644 index 000000000..16bc68db9 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-0.orc differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-1.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-1.orc new file mode 100644 index 000000000..fa46994ff Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-1.orc differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-2e50e625-15ac-4994-a320-064d8e34d028-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-2e50e625-15ac-4994-a320-064d8e34d028-0 new file mode 100644 index 000000000..77fc5c530 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-2e50e625-15ac-4994-a320-064d8e34d028-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-5890ac3e-74e0-4d3c-bddd-542207d3a13f-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-5890ac3e-74e0-4d3c-bddd-542207d3a13f-0 new file mode 100644 index 000000000..6b2aaa764 --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-5890ac3e-74e0-4d3c-bddd-542207d3a13f-0 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-7d01de51-d440-42c2-a2ac-f8b89d8e9d36-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-7d01de51-d440-42c2-a2ac-f8b89d8e9d36-0 new file mode 100644 index 000000000..07832286c Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-7d01de51-d440-42c2-a2ac-f8b89d8e9d36-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-a3ae7e50-6220-4663-89de-4b9b31f41a39-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-a3ae7e50-6220-4663-89de-4b9b31f41a39-0 new file mode 100644 index 000000000..c9776973c Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-a3ae7e50-6220-4663-89de-4b9b31f41a39-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-c940705f-8d27-40f3-a9ba-a896eeecd29a-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-c940705f-8d27-40f3-a9ba-a896eeecd29a-0 new file mode 100644 index 000000000..025c6a081 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-c940705f-8d27-40f3-a9ba-a896eeecd29a-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-456ef7ae-b9e5-4e2a-bc33-b6afade33eac-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-456ef7ae-b9e5-4e2a-bc33-b6afade33eac-0 new file mode 100644 index 000000000..425c4a4d3 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-456ef7ae-b9e5-4e2a-bc33-b6afade33eac-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-9dbada10-b497-4ef7-bf69-a3bdb5959a21-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-9dbada10-b497-4ef7-bf69-a3bdb5959a21-0 new file mode 100644 index 000000000..0157b8125 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-9dbada10-b497-4ef7-bf69-a3bdb5959a21-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-cbbe0867-55db-48b3-a617-01a8b5ea66e8-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-cbbe0867-55db-48b3-a617-01a8b5ea66e8-0 new file mode 100644 index 000000000..faeddedfe Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-cbbe0867-55db-48b3-a617-01a8b5ea66e8-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-5db2d0f1-0070-4dd6-b135-b62ce82c1502-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-5db2d0f1-0070-4dd6-b135-b62ce82c1502-0 new file mode 100644 index 000000000..6d780a6aa Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-5db2d0f1-0070-4dd6-b135-b62ce82c1502-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-8175e923-c512-4513-9325-bbf4a875b2d9-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-8175e923-c512-4513-9325-bbf4a875b2d9-0 new file mode 100644 index 000000000..8997bc8ff Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-8175e923-c512-4513-9325-bbf4a875b2d9-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-999f9f03-1b9c-4967-bda2-441ecce6bc77-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-999f9f03-1b9c-4967-bda2-441ecce6bc77-0 new file mode 100644 index 000000000..5f98e5927 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-999f9f03-1b9c-4967-bda2-441ecce6bc77-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-c08df75e-4def-48df-b351-b5fe73087467-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-c08df75e-4def-48df-b351-b5fe73087467-0 new file mode 100644 index 000000000..b0e1dd9f4 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-c08df75e-4def-48df-b351-b5fe73087467-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-d102f891-c912-4cff-92f7-f5765f8d2248-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-d102f891-c912-4cff-92f7-f5765f8d2248-0 new file mode 100644 index 000000000..eb44f0c8e Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-d102f891-c912-4cff-92f7-f5765f8d2248-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-0 new file mode 100644 index 000000000..cbe9bbad3 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-1 new file mode 100644 index 000000000..4277b15b4 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-1 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-0 new file mode 100644 index 000000000..0025b4cfb Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-1 new file mode 100644 index 000000000..8646a801e Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-1 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-0 new file mode 100644 index 000000000..7d34ea310 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-1 new file mode 100644 index 000000000..53c972baf Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-1 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-0 new file mode 100644 index 000000000..a505feada Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-1 new file mode 100644 index 000000000..7efc617f1 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-1 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-0 new file mode 100644 index 000000000..bc78e37b9 Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-0 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-1 new file mode 100644 index 000000000..625a0f54a Binary files /dev/null and b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-1 differ diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 new file mode 100644 index 000000000..e11b68d77 --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 @@ -0,0 +1,32 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "score", + "type" : "INT" + }, { + "id" : 2, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ "id" ], + "options" : { + "bucket" : "1", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "orc", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866469063 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST new file mode 100644 index 000000000..7813681f5 --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 new file mode 100644 index 000000000..c992e494f --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "uuid" : "f6973129-e5fa-490b-9f56-92c8914efae5", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-1", + "deltaManifestListSize" : 1113, + "commitUser" : "f9356237-8f3f-4cd7-8b48-5e9868517db0", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866469281, + "totalRecordCount" : 2000, + "deltaRecordCount" : 2000, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 new file mode 100644 index 000000000..e8ed3e3aa --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "64f64df3-ede4-4526-a23d-a296ea5032a8", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-0", + "baseManifestListSize" : 1113, + "deltaManifestList" : "manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-1", + "deltaManifestListSize" : 1115, + "indexManifest" : "index-manifest-9dbada10-b497-4ef7-bf69-a3bdb5959a21-0", + "commitUser" : "7b20f907-0770-439f-9f92-b2488a32aaf8", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469362, + "totalRecordCount" : 2000, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 new file mode 100644 index 000000000..2b5b2ccbc --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "c2846a82-ce6b-4287-90f9-13cec431b7d5", + "id" : 3, + "schemaId" : 0, + "baseManifestList" : "manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-0", + "baseManifestListSize" : 1148, + "deltaManifestList" : "manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-1", + "deltaManifestListSize" : 1117, + "indexManifest" : "index-manifest-9dbada10-b497-4ef7-bf69-a3bdb5959a21-0", + "commitUser" : "ceeaf629-1e42-43c1-bf0a-7c5ca9aa95be", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866469394, + "totalRecordCount" : 2002, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 new file mode 100644 index 000000000..e9ce1d153 --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "b4201f1c-9414-4388-825e-6397d2c78d3a", + "id" : 4, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-0", + "baseManifestListSize" : 1184, + "deltaManifestList" : "manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-1", + "deltaManifestListSize" : 1117, + "indexManifest" : "index-manifest-456ef7ae-b9e5-4e2a-bc33-b6afade33eac-0", + "commitUser" : "eeac6851-7421-426c-bdfe-fb2affcfd8d7", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469447, + "totalRecordCount" : 2003, + "deltaRecordCount" : 1, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 new file mode 100644 index 000000000..8af80a1a2 --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "3f18b21b-6f35-4311-b71e-beb3ea4451c6", + "id" : 5, + "schemaId" : 0, + "baseManifestList" : "manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-0", + "baseManifestListSize" : 1218, + "deltaManifestList" : "manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-1", + "deltaManifestListSize" : 1114, + "indexManifest" : "index-manifest-cbbe0867-55db-48b3-a617-01a8b5ea66e8-0", + "commitUser" : "688ae21f-38b6-4ca3-b074-4b88af33bf1d", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469519, + "totalRecordCount" : 2001, + "deltaRecordCount" : -2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README new file mode 100644 index 000000000..6bbb500d7 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README @@ -0,0 +1,21 @@ +pt:int id:int score:int tag:string +primary key: pt,id +partition key: pt +bucket count: 2 (both buckets populated: [0, 1]) + +Generated by Apache Paimon Java release-2.0.0. +file format: orc +deletion-vectors.enabled: true +deletion-vectors.merge-on-read: false +write-only: true (explicit fixture commits produce snapshots 2, 4, 5) +primary-key btree index: score + +Snapshot semantics: +snapshot-1 APPEND: pt=1 and pt=2, ids 1..100 in each; score=(pt*100+id)%10; even tag=keep, odd tag=drop; 200 rows. +snapshot-2 COMPACT: full compaction of every populated partition/bucket builds source-backed score BTree payloads; 200 rows. +tag fallback-base and branch fallback are created from snapshot-2; the branch remains at its 200 indexed rows; it is used to validate fallback-branch TableRead routing. +snapshot-3 APPEND: add (id=101,score=0,tag=keep) to both partitions; indexed compacted files and visible unindexed APPEND-source files coexist; 202 rows. +snapshot-4 COMPACT: delete (pt=1,id=10), update (pt=2,id=10) to score=88/tag=updated, then lookup compact in one fixture commit; deletion vectors are present; 201 rows. +snapshot-5 COMPACT: full compaction rebuilds all partition/bucket BTree groups; 201 rows. + +Fixture construction note: with deletion vectors enabled and merge-on-read disabled, Java and C++ batch scans skip literal level-0 files. Snapshot 3 therefore promotes Java-prepared files to the maximum manifest level while retaining FileSource.APPEND. The source-backed PK BTree policy indexes only COMPACT files above level 0, making these files visible but deliberately unindexed. Snapshot 4 publishes the Java-prepared compact outputs and index changes while omitting their transient, never-active append inputs. These are test-fixture constructions, not ordinary end-user write workflows. diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 new file mode 100644 index 000000000..ff018d173 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "pt", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 2, + "name" : "score", + "type" : "INT" + }, { + "id" : 3, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 3, + "partitionKeys" : [ "pt" ], + "primaryKeys" : [ "pt", "id" ], + "options" : { + "bucket" : "2", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "orc", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866469580 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 new file mode 100644 index 000000000..106e9b363 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "40f471de-0c7c-4a98-80d3-f36f726ae007", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0", + "baseManifestListSize" : 1118, + "deltaManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0", + "commitUser" : "f8c33e9f-0bff-4bdc-9b90-67e951ad7421", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469757, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base new file mode 100644 index 000000000..106e9b363 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "40f471de-0c7c-4a98-80d3-f36f726ae007", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0", + "baseManifestListSize" : 1118, + "deltaManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0", + "commitUser" : "f8c33e9f-0bff-4bdc-9b90-67e951ad7421", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469757, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-0 new file mode 100644 index 000000000..6b2aaa764 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-0 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-1 new file mode 100644 index 000000000..6b2aaa764 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-1 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-0 new file mode 100644 index 000000000..01345b017 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-1 new file mode 100644 index 000000000..aab0e6e31 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-1 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-2 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-2 new file mode 100644 index 000000000..c97470bd4 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-2 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-3 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-3 new file mode 100644 index 000000000..aab0e6e31 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-3 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-986b5fda-9721-4e64-9a1a-36b9bf2ad357-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-986b5fda-9721-4e64-9a1a-36b9bf2ad357-0 new file mode 100644 index 000000000..484c9f7b7 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-986b5fda-9721-4e64-9a1a-36b9bf2ad357-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-0 new file mode 100644 index 000000000..a221074bd Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-1 new file mode 100644 index 000000000..aab0e6e31 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-1 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-2 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-2 new file mode 100644 index 000000000..a221074bd Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-2 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-3 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-3 new file mode 100644 index 000000000..aab0e6e31 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-3 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-0 new file mode 100644 index 000000000..26cce0f82 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-1 new file mode 100644 index 000000000..26cce0f82 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-1 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0 new file mode 100644 index 000000000..d9f378eb6 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-55558948-e419-4c96-912a-f7d4b1d25af6-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-55558948-e419-4c96-912a-f7d4b1d25af6-0 new file mode 100644 index 000000000..cb453879d Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-55558948-e419-4c96-912a-f7d4b1d25af6-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-cb21ca4f-0678-48b5-b2ba-f797e7dedab4-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-cb21ca4f-0678-48b5-b2ba-f797e7dedab4-0 new file mode 100644 index 000000000..665086be6 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-cb21ca4f-0678-48b5-b2ba-f797e7dedab4-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-1b596e7e-4a38-4963-8e62-1c50fb11c73e-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-1b596e7e-4a38-4963-8e62-1c50fb11c73e-0 new file mode 100644 index 000000000..2284d5668 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-1b596e7e-4a38-4963-8e62-1c50fb11c73e-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-24630e00-34bd-45e8-9ffc-e1b061217359-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-24630e00-34bd-45e8-9ffc-e1b061217359-0 new file mode 100644 index 000000000..ee0801244 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-24630e00-34bd-45e8-9ffc-e1b061217359-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8f4fff72-e24d-4ac8-b2fe-a39b2ba3d528-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8f4fff72-e24d-4ac8-b2fe-a39b2ba3d528-0 new file mode 100644 index 000000000..1bee4e5fc Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8f4fff72-e24d-4ac8-b2fe-a39b2ba3d528-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-b3d6bc4a-6665-49ea-9d2e-0f2ac5fd142c-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-b3d6bc4a-6665-49ea-9d2e-0f2ac5fd142c-0 new file mode 100644 index 000000000..514ea1bc1 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-b3d6bc4a-6665-49ea-9d2e-0f2ac5fd142c-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e2a0a966-17b2-4269-9362-2bde27472a88-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e2a0a966-17b2-4269-9362-2bde27472a88-0 new file mode 100644 index 000000000..fdc48d3f6 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e2a0a966-17b2-4269-9362-2bde27472a88-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-0 new file mode 100644 index 000000000..3772c770f Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-1 new file mode 100644 index 000000000..0265b6cea Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-1 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-0 new file mode 100644 index 000000000..bf901448d Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-1 new file mode 100644 index 000000000..498dddf11 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-1 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-0 new file mode 100644 index 000000000..e8696c2d1 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-1 new file mode 100644 index 000000000..c9a297ea3 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-1 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0 new file mode 100644 index 000000000..292cd2e71 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1 new file mode 100644 index 000000000..40470d951 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-0 new file mode 100644 index 000000000..be199713e Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-0 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-1 new file mode 100644 index 000000000..84a37dffb Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-1 differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0038846e-e990-4367-a488-f390312bae65-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0038846e-e990-4367-a488-f390312bae65-0.orc new file mode 100644 index 000000000..64d577533 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0038846e-e990-4367-a488-f390312bae65-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0d30be46-8b96-4020-b2fc-ec814be128c2-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0d30be46-8b96-4020-b2fc-ec814be128c2-0.orc new file mode 100644 index 000000000..2e0462494 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0d30be46-8b96-4020-b2fc-ec814be128c2-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-76bfec84-0851-4a14-8c8e-5e9f54c18ef2-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-76bfec84-0851-4a14-8c8e-5e9f54c18ef2-0.orc new file mode 100644 index 000000000..ab399d823 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-76bfec84-0851-4a14-8c8e-5e9f54c18ef2-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-a6c5a536-5e45-4d9b-baae-742f93ee670d-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-a6c5a536-5e45-4d9b-baae-742f93ee670d-0.orc new file mode 100644 index 000000000..205ccfafa Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-a6c5a536-5e45-4d9b-baae-742f93ee670d-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-a6c5a536-5e45-4d9b-baae-742f93ee670d-1.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-a6c5a536-5e45-4d9b-baae-742f93ee670d-1.orc new file mode 100644 index 000000000..64d577533 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-a6c5a536-5e45-4d9b-baae-742f93ee670d-1.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-e32042c3-9e41-4cd7-8bcb-59e0944b3803-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-e32042c3-9e41-4cd7-8bcb-59e0944b3803-0.orc new file mode 100644 index 000000000..ab399d823 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-e32042c3-9e41-4cd7-8bcb-59e0944b3803-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-1245ceb5-2933-4397-89f9-5363b9a46498-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-1245ceb5-2933-4397-89f9-5363b9a46498-0.orc new file mode 100644 index 000000000..b49d67dc0 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-1245ceb5-2933-4397-89f9-5363b9a46498-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-dc40f35c-9fff-40a3-a08b-38898a7538a9-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-dc40f35c-9fff-40a3-a08b-38898a7538a9-0.orc new file mode 100644 index 000000000..b49d67dc0 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-dc40f35c-9fff-40a3-a08b-38898a7538a9-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-f818ca94-495e-4146-a8ad-35a4474f5058-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-f818ca94-495e-4146-a8ad-35a4474f5058-0.orc new file mode 100644 index 000000000..b49d67dc0 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-f818ca94-495e-4146-a8ad-35a4474f5058-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-3940b910-9cc6-44bd-8a34-ee00e35f4977-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-3940b910-9cc6-44bd-8a34-ee00e35f4977-0.orc new file mode 100644 index 000000000..fb4c9d712 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-3940b910-9cc6-44bd-8a34-ee00e35f4977-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-7e663126-95fd-402d-b971-80372baa3fe7-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-7e663126-95fd-402d-b971-80372baa3fe7-0.orc new file mode 100644 index 000000000..f41c03024 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-7e663126-95fd-402d-b971-80372baa3fe7-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-835df91b-8a15-4ee4-b118-4a4f5429c2f1-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-835df91b-8a15-4ee4-b118-4a4f5429c2f1-0.orc new file mode 100644 index 000000000..f41c03024 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-835df91b-8a15-4ee4-b118-4a4f5429c2f1-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-91617322-2bb2-4ad1-a445-e1dd850d2139-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-91617322-2bb2-4ad1-a445-e1dd850d2139-0.orc new file mode 100644 index 000000000..a8da082df Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-91617322-2bb2-4ad1-a445-e1dd850d2139-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-a2e76792-9576-4276-8bc0-8a76cbeb4f9c-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-a2e76792-9576-4276-8bc0-8a76cbeb4f9c-0.orc new file mode 100644 index 000000000..29c881b53 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-a2e76792-9576-4276-8bc0-8a76cbeb4f9c-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-dcc6ed0d-38b1-4447-b3e6-5b1788fc948a-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-dcc6ed0d-38b1-4447-b3e6-5b1788fc948a-0.orc new file mode 100644 index 000000000..fb4c9d712 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-dcc6ed0d-38b1-4447-b3e6-5b1788fc948a-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-2a0e2e33-9592-4be7-9d68-d7f57c9d735c-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-2a0e2e33-9592-4be7-9d68-d7f57c9d735c-0.orc new file mode 100644 index 000000000..df9e368a0 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-2a0e2e33-9592-4be7-9d68-d7f57c9d735c-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1dcd608-70b5-4d7f-afd2-676c97fb9fac-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1dcd608-70b5-4d7f-afd2-676c97fb9fac-0.orc new file mode 100644 index 000000000..df9e368a0 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1dcd608-70b5-4d7f-afd2-676c97fb9fac-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-d9aabc21-f287-48ae-baea-5b6d70640f0d-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-d9aabc21-f287-48ae-baea-5b6d70640f0d-0.orc new file mode 100644 index 000000000..df9e368a0 Binary files /dev/null and b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-d9aabc21-f287-48ae-baea-5b6d70640f0d-0.orc differ diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 new file mode 100644 index 000000000..ff018d173 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "pt", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 2, + "name" : "score", + "type" : "INT" + }, { + "id" : 3, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 3, + "partitionKeys" : [ "pt" ], + "primaryKeys" : [ "pt", "id" ], + "options" : { + "bucket" : "2", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "orc", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866469580 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST new file mode 100644 index 000000000..7813681f5 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 new file mode 100644 index 000000000..23aca360d --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "uuid" : "2337084c-4fae-4c87-869c-1794e79f560c", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-1", + "deltaManifestListSize" : 1118, + "commitUser" : "75a0f162-3926-40b4-b6ec-aa5da579c3c3", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866469655, + "totalRecordCount" : 200, + "deltaRecordCount" : 200, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 new file mode 100644 index 000000000..106e9b363 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "40f471de-0c7c-4a98-80d3-f36f726ae007", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0", + "baseManifestListSize" : 1118, + "deltaManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0", + "commitUser" : "f8c33e9f-0bff-4bdc-9b90-67e951ad7421", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469757, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 new file mode 100644 index 000000000..c1aa92c65 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "95913adb-0e78-48df-ab27-3566a670199c", + "id" : 3, + "schemaId" : 0, + "baseManifestList" : "manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-0", + "baseManifestListSize" : 1159, + "deltaManifestList" : "manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0", + "commitUser" : "c31fe56e-a1d2-4a59-aeb8-ce3be0915349", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866469817, + "totalRecordCount" : 202, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 new file mode 100644 index 000000000..136e05854 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "a0fa8d3a-44e1-4b98-bc68-c8b89bf3da42", + "id" : 4, + "schemaId" : 0, + "baseManifestList" : "manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-0", + "baseManifestListSize" : 1196, + "deltaManifestList" : "manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-1", + "deltaManifestListSize" : 1122, + "indexManifest" : "index-manifest-cb21ca4f-0678-48b5-b2ba-f797e7dedab4-0", + "commitUser" : "07771156-18a8-4384-b947-4412c4bf53e6", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469903, + "totalRecordCount" : 203, + "deltaRecordCount" : 1, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 new file mode 100644 index 000000000..3ad236ffc --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "e255e37c-142e-4888-812b-05a76e52f996", + "id" : 5, + "schemaId" : 0, + "baseManifestList" : "manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-0", + "baseManifestListSize" : 1234, + "deltaManifestList" : "manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-1", + "deltaManifestListSize" : 1126, + "indexManifest" : "index-manifest-55558948-e419-4c96-912a-f7d4b1d25af6-0", + "commitUser" : "3a3579f6-c63b-4621-b1dc-c08a781f0181", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469992, + "totalRecordCount" : 201, + "deltaRecordCount" : -2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base new file mode 100644 index 000000000..106e9b363 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "40f471de-0c7c-4a98-80d3-f36f726ae007", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0", + "baseManifestListSize" : 1118, + "deltaManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0", + "commitUser" : "f8c33e9f-0bff-4bdc-9b90-67e951ad7421", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469757, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/README b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/README new file mode 100644 index 000000000..43ba26ffb --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/README @@ -0,0 +1,24 @@ +id:int score:int tag:string +primary key: id +no partition key +bucket count: 1 + +Generated by Apache Paimon Java release-2.0.0. +file format: parquet +deletion-vectors.enabled: true +deletion-vectors.merge-on-read: false +write-only: true (explicit fixture commits produce snapshots 2, 4, 5) +primary-key btree index: score +target-file-size: 8 kb +compaction.force-rewrite-all-files: true + +Snapshot semantics: +snapshot-1 APPEND: ids 1..2000; score=id%10; even tag=keep, odd tag=drop. +snapshot-2 COMPACT: full compaction builds source-backed score BTree payloads; source-file counts per payload are [2]; 2000 rows. +snapshot-3 APPEND: insert (2001,0,keep) and (2002,5,late_drop); indexed compacted files and a visible unindexed APPEND-source file coexist; 2002 rows. +snapshot-4 COMPACT: delete id=10 and update id=20 to (77,updated), then lookup compact in one fixture commit; deletion vectors coexist with indexed files; 2001 rows. +snapshot-5 COMPACT: full compaction rebuilds the BTree source groups; 2001 rows. + +Fixture construction note: with deletion vectors enabled and merge-on-read disabled, Java and C++ batch scans skip literal level-0 files. Snapshot 3 therefore promotes Java-prepared files to the maximum manifest level while retaining FileSource.APPEND. The source-backed PK BTree policy indexes only COMPACT files above level 0, making these files visible but deliberately unindexed. Snapshot 4 publishes the Java-prepared compact outputs and index changes while omitting their transient, never-active append inputs. These are test-fixture constructions, not ordinary end-user write workflows. + +At snapshots 4 and 5, predicate score=0 AND id<=1500 returns ids 30,40,...,1500; id=10 is deleted and id=20 has score 77. diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-0caae41e-5153-4d29-8a48-dbbdf015f77a-0.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-0caae41e-5153-4d29-8a48-dbbdf015f77a-0.parquet new file mode 100644 index 000000000..17dcdd3d0 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-0caae41e-5153-4d29-8a48-dbbdf015f77a-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-14f3c2d2-86bc-4305-8ddc-3d7587b8e8ac-0.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-14f3c2d2-86bc-4305-8ddc-3d7587b8e8ac-0.parquet new file mode 100644 index 000000000..0564c8255 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-14f3c2d2-86bc-4305-8ddc-3d7587b8e8ac-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-1599b187-f60c-439b-841c-b0fa0a78da4f-0.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-1599b187-f60c-439b-841c-b0fa0a78da4f-0.parquet new file mode 100644 index 000000000..00afd6dd8 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-1599b187-f60c-439b-841c-b0fa0a78da4f-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-0.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-0.parquet new file mode 100644 index 000000000..8838eb1eb Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-1.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-1.parquet new file mode 100644 index 000000000..4403ca09d Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-1.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-0.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-0.parquet new file mode 100644 index 000000000..8838eb1eb Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-1.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-1.parquet new file mode 100644 index 000000000..4403ca09d Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-1.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-0.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-0.parquet new file mode 100644 index 000000000..f0574e34b Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-1.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-1.parquet new file mode 100644 index 000000000..4403ca09d Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-1.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-2.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-2.parquet new file mode 100644 index 000000000..17dcdd3d0 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-2.parquet differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-1e092fe2-2c76-4682-bfec-50ebecd3957a-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-1e092fe2-2c76-4682-bfec-50ebecd3957a-0 new file mode 100644 index 000000000..025c6a081 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-1e092fe2-2c76-4682-bfec-50ebecd3957a-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-360cfb9c-6646-44cb-aec5-d156a379b4d2-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-360cfb9c-6646-44cb-aec5-d156a379b4d2-0 new file mode 100644 index 000000000..77fc5c530 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-360cfb9c-6646-44cb-aec5-d156a379b4d2-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-5561631d-80ef-45bc-8b87-3d1a2876816d-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-5561631d-80ef-45bc-8b87-3d1a2876816d-0 new file mode 100644 index 000000000..07832286c Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-5561631d-80ef-45bc-8b87-3d1a2876816d-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-806143e3-9271-4934-9ca5-636a815c46a0-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-806143e3-9271-4934-9ca5-636a815c46a0-0 new file mode 100644 index 000000000..c9776973c Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-806143e3-9271-4934-9ca5-636a815c46a0-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-a353b7ca-671b-4423-bff5-fd2888db7536-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-a353b7ca-671b-4423-bff5-fd2888db7536-0 new file mode 100644 index 000000000..6b2aaa764 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-a353b7ca-671b-4423-bff5-fd2888db7536-0 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0 new file mode 100644 index 000000000..087944291 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-c09d3b21-6083-4881-a6f6-b0ecc37dac22-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-c09d3b21-6083-4881-a6f6-b0ecc37dac22-0 new file mode 100644 index 000000000..8c6909a73 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-c09d3b21-6083-4881-a6f6-b0ecc37dac22-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-d52df85e-6f91-497a-8edd-df92bcc3e772-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-d52df85e-6f91-497a-8edd-df92bcc3e772-0 new file mode 100644 index 000000000..604bac2f3 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-d52df85e-6f91-497a-8edd-df92bcc3e772-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-425d391e-cf9c-4770-9241-09facc794f72-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-425d391e-cf9c-4770-9241-09facc794f72-0 new file mode 100644 index 000000000..2c7c92e21 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-425d391e-cf9c-4770-9241-09facc794f72-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-5b55486f-5a96-408a-9079-c51b68c5dec9-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-5b55486f-5a96-408a-9079-c51b68c5dec9-0 new file mode 100644 index 000000000..3b7b969a4 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-5b55486f-5a96-408a-9079-c51b68c5dec9-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-97c08d9a-9e64-4d25-8e37-3350e3eba2aa-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-97c08d9a-9e64-4d25-8e37-3350e3eba2aa-0 new file mode 100644 index 000000000..3eddd2b0a Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-97c08d9a-9e64-4d25-8e37-3350e3eba2aa-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-a01531dd-94c8-4b58-b844-b0e1f26a464c-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-a01531dd-94c8-4b58-b844-b0e1f26a464c-0 new file mode 100644 index 000000000..232d818a0 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-a01531dd-94c8-4b58-b844-b0e1f26a464c-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-dc356a38-74d8-41b5-a7b8-270860bf71ad-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-dc356a38-74d8-41b5-a7b8-270860bf71ad-0 new file mode 100644 index 000000000..2b0ba83c6 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-dc356a38-74d8-41b5-a7b8-270860bf71ad-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-0 new file mode 100644 index 000000000..2f624855d Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-1 new file mode 100644 index 000000000..eab85c7ee Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-1 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-0 new file mode 100644 index 000000000..c95885993 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-1 new file mode 100644 index 000000000..f5048a337 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-1 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-0 new file mode 100644 index 000000000..ba35d107e Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-1 new file mode 100644 index 000000000..557e57b79 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-1 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-0 new file mode 100644 index 000000000..b938c6655 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-1 new file mode 100644 index 000000000..233f82c18 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-1 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-0 new file mode 100644 index 000000000..f751ffc02 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-0 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-1 new file mode 100644 index 000000000..469fa0de9 Binary files /dev/null and b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-1 differ diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 new file mode 100644 index 000000000..072417e06 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 @@ -0,0 +1,32 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "score", + "type" : "INT" + }, { + "id" : 2, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ "id" ], + "options" : { + "bucket" : "1", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "parquet", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866466351 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST new file mode 100644 index 000000000..7813681f5 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 new file mode 100644 index 000000000..65ae67734 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "uuid" : "67d8f6d2-d970-4968-9e30-0065cad82aaa", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-1", + "deltaManifestListSize" : 1110, + "commitUser" : "d042b1e3-b731-4f72-ad18-29291dfd2e63", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866467778, + "totalRecordCount" : 2000, + "deltaRecordCount" : 2000, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 new file mode 100644 index 000000000..a2257b73e --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "2cf4b059-0a02-499a-af85-db54b6af0d2d", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-0", + "baseManifestListSize" : 1110, + "deltaManifestList" : "manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-1", + "deltaManifestListSize" : 1112, + "indexManifest" : "index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0", + "commitUser" : "9e2ab339-ce47-48af-83e0-2ed698c964c3", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468252, + "totalRecordCount" : 2000, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 new file mode 100644 index 000000000..79b8ead14 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "2141b3d7-41f2-4cc6-8378-bf7022b59254", + "id" : 3, + "schemaId" : 0, + "baseManifestList" : "manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-0", + "baseManifestListSize" : 1149, + "deltaManifestList" : "manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-1", + "deltaManifestListSize" : 1117, + "indexManifest" : "index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0", + "commitUser" : "61bb613e-40c7-4ae2-a590-35ae5358d134", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866468287, + "totalRecordCount" : 2002, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 new file mode 100644 index 000000000..65ab45800 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "7e9696ad-f881-479a-9422-5b61f34d0688", + "id" : 4, + "schemaId" : 0, + "baseManifestList" : "manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-0", + "baseManifestListSize" : 1186, + "deltaManifestList" : "manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-1", + "deltaManifestListSize" : 1117, + "indexManifest" : "index-manifest-c09d3b21-6083-4881-a6f6-b0ecc37dac22-0", + "commitUser" : "c8e2d8ed-8282-4403-a00b-3c3908227c44", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468364, + "totalRecordCount" : 2003, + "deltaRecordCount" : 1, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 new file mode 100644 index 000000000..a91d0a29f --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "e5a70fc9-5b16-4cbc-9707-b3ed5b83484a", + "id" : 5, + "schemaId" : 0, + "baseManifestList" : "manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-0", + "baseManifestListSize" : 1219, + "deltaManifestList" : "manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-1", + "deltaManifestListSize" : 1117, + "indexManifest" : "index-manifest-d52df85e-6f91-497a-8edd-df92bcc3e772-0", + "commitUser" : "69965629-0d92-4ff4-a2b3-673f8ebb74f6", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468584, + "totalRecordCount" : 2001, + "deltaRecordCount" : -2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README new file mode 100644 index 000000000..7318572fb --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README @@ -0,0 +1,21 @@ +pt:int id:int score:int tag:string +primary key: pt,id +partition key: pt +bucket count: 2 (both buckets populated: [0, 1]) + +Generated by Apache Paimon Java release-2.0.0. +file format: parquet +deletion-vectors.enabled: true +deletion-vectors.merge-on-read: false +write-only: true (explicit fixture commits produce snapshots 2, 4, 5) +primary-key btree index: score + +Snapshot semantics: +snapshot-1 APPEND: pt=1 and pt=2, ids 1..100 in each; score=(pt*100+id)%10; even tag=keep, odd tag=drop; 200 rows. +snapshot-2 COMPACT: full compaction of every populated partition/bucket builds source-backed score BTree payloads; 200 rows. +tag fallback-base and branch fallback are created from snapshot-2; the branch remains at its 200 indexed rows; it is used to validate fallback-branch TableRead routing. +snapshot-3 APPEND: add (id=101,score=0,tag=keep) to both partitions; indexed compacted files and visible unindexed APPEND-source files coexist; 202 rows. +snapshot-4 COMPACT: delete (pt=1,id=10), update (pt=2,id=10) to score=88/tag=updated, then lookup compact in one fixture commit; deletion vectors are present; 201 rows. +snapshot-5 COMPACT: full compaction rebuilds all partition/bucket BTree groups; 201 rows. + +Fixture construction note: with deletion vectors enabled and merge-on-read disabled, Java and C++ batch scans skip literal level-0 files. Snapshot 3 therefore promotes Java-prepared files to the maximum manifest level while retaining FileSource.APPEND. The source-backed PK BTree policy indexes only COMPACT files above level 0, making these files visible but deliberately unindexed. Snapshot 4 publishes the Java-prepared compact outputs and index changes while omitting their transient, never-active append inputs. These are test-fixture constructions, not ordinary end-user write workflows. diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 new file mode 100644 index 000000000..ddc738c05 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "pt", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 2, + "name" : "score", + "type" : "INT" + }, { + "id" : 3, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 3, + "partitionKeys" : [ "pt" ], + "primaryKeys" : [ "pt", "id" ], + "options" : { + "bucket" : "2", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "parquet", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866468700 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 new file mode 100644 index 000000000..c4ef63281 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "1f5e235e-3e28-4ab4-812f-6a1e4ca39056", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "b2b00e43-1a8e-4cb1-84da-0f003902e88b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468867, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base new file mode 100644 index 000000000..c4ef63281 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "1f5e235e-3e28-4ab4-812f-6a1e4ca39056", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "b2b00e43-1a8e-4cb1-84da-0f003902e88b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468867, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-0 new file mode 100644 index 000000000..26cce0f82 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-1 new file mode 100644 index 000000000..26cce0f82 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-1 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-523fefd0-769f-46f8-8b58-957065ad7e20-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-523fefd0-769f-46f8-8b58-957065ad7e20-0 new file mode 100644 index 000000000..6b2aaa764 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-523fefd0-769f-46f8-8b58-957065ad7e20-0 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-523fefd0-769f-46f8-8b58-957065ad7e20-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-523fefd0-769f-46f8-8b58-957065ad7e20-1 new file mode 100644 index 000000000..6b2aaa764 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-523fefd0-769f-46f8-8b58-957065ad7e20-1 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-622c2733-874d-4ebb-b076-500a21ad0432-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-622c2733-874d-4ebb-b076-500a21ad0432-0 new file mode 100644 index 000000000..484c9f7b7 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-622c2733-874d-4ebb-b076-500a21ad0432-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-0 new file mode 100644 index 000000000..a221074bd Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-1 new file mode 100644 index 000000000..aab0e6e31 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-1 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-2 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-2 new file mode 100644 index 000000000..a221074bd Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-2 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-3 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-3 new file mode 100644 index 000000000..aab0e6e31 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-3 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-0 new file mode 100644 index 000000000..01345b017 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-1 new file mode 100644 index 000000000..aab0e6e31 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-1 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-2 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-2 new file mode 100644 index 000000000..c97470bd4 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-2 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-3 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-3 new file mode 100644 index 000000000..aab0e6e31 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-3 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-11be7772-670d-4dc1-adbb-fd8413f2779d-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-11be7772-670d-4dc1-adbb-fd8413f2779d-0 new file mode 100644 index 000000000..2f1aefc19 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-11be7772-670d-4dc1-adbb-fd8413f2779d-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0 new file mode 100644 index 000000000..2b494589f Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-a9b54c7b-7468-4a64-97ee-32adae28681e-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-a9b54c7b-7468-4a64-97ee-32adae28681e-0 new file mode 100644 index 000000000..98f226fb9 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-a9b54c7b-7468-4a64-97ee-32adae28681e-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8727f0fe-6af1-4c2f-8560-c38dd0a81ec0-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8727f0fe-6af1-4c2f-8560-c38dd0a81ec0-0 new file mode 100644 index 000000000..c09730c36 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8727f0fe-6af1-4c2f-8560-c38dd0a81ec0-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-bd400d22-3691-47bb-8262-753eb1a8c2c4-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-bd400d22-3691-47bb-8262-753eb1a8c2c4-0 new file mode 100644 index 000000000..d055bf247 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-bd400d22-3691-47bb-8262-753eb1a8c2c4-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-dd531b47-e547-4b27-b799-09842a4b026d-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-dd531b47-e547-4b27-b799-09842a4b026d-0 new file mode 100644 index 000000000..97490cde9 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-dd531b47-e547-4b27-b799-09842a4b026d-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e6ecca13-a405-472b-b1a4-77de887ae406-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e6ecca13-a405-472b-b1a4-77de887ae406-0 new file mode 100644 index 000000000..db786b045 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e6ecca13-a405-472b-b1a4-77de887ae406-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-f8dc9dce-972e-41a4-b6dc-a8eae92847f4-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-f8dc9dce-972e-41a4-b6dc-a8eae92847f4-0 new file mode 100644 index 000000000..8b66e74c5 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-f8dc9dce-972e-41a4-b6dc-a8eae92847f4-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-0 new file mode 100644 index 000000000..d7f7ae1d0 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-1 new file mode 100644 index 000000000..8ca6dced7 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-1 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0 new file mode 100644 index 000000000..e87a46531 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1 new file mode 100644 index 000000000..ec68268f5 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-0 new file mode 100644 index 000000000..1212f12e5 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-1 new file mode 100644 index 000000000..7e1d22226 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-1 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-0 new file mode 100644 index 000000000..fde5da56a Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-1 new file mode 100644 index 000000000..b9cd1bb12 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-1 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-0 new file mode 100644 index 000000000..de2e7a181 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-0 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-1 new file mode 100644 index 000000000..c2665104d Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-1 differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-2826504c-fd4a-4ef7-8f63-724e727201e9-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-2826504c-fd4a-4ef7-8f63-724e727201e9-0.parquet new file mode 100644 index 000000000..c2b4891d0 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-2826504c-fd4a-4ef7-8f63-724e727201e9-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-0.parquet new file mode 100644 index 000000000..81cdd6d60 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-1.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-1.parquet new file mode 100644 index 000000000..c2b4891d0 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-1.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-6444b55f-03c4-4cdf-829a-1cd0c9d670ce-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-6444b55f-03c4-4cdf-829a-1cd0c9d670ce-0.parquet new file mode 100644 index 000000000..4fc0fb7f8 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-6444b55f-03c4-4cdf-829a-1cd0c9d670ce-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-c469efe8-01bf-4123-a9f6-e9505701ffba-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-c469efe8-01bf-4123-a9f6-e9505701ffba-0.parquet new file mode 100644 index 000000000..3ce73a274 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-c469efe8-01bf-4123-a9f6-e9505701ffba-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-cda2ddb8-fcd1-4298-a4a8-4778ab7aa7a6-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-cda2ddb8-fcd1-4298-a4a8-4778ab7aa7a6-0.parquet new file mode 100644 index 000000000..3ce73a274 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-cda2ddb8-fcd1-4298-a4a8-4778ab7aa7a6-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-0f061bb7-3867-4575-a7f3-a649acff590b-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-0f061bb7-3867-4575-a7f3-a649acff590b-0.parquet new file mode 100644 index 000000000..0099628a4 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-0f061bb7-3867-4575-a7f3-a649acff590b-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-42b50aa8-9cf7-409d-a011-f7d6a0cad4a8-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-42b50aa8-9cf7-409d-a011-f7d6a0cad4a8-0.parquet new file mode 100644 index 000000000..0099628a4 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-42b50aa8-9cf7-409d-a011-f7d6a0cad4a8-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-d1e79d3b-bb1f-4afe-a6b3-8cc85490af74-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-d1e79d3b-bb1f-4afe-a6b3-8cc85490af74-0.parquet new file mode 100644 index 000000000..0099628a4 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-d1e79d3b-bb1f-4afe-a6b3-8cc85490af74-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-043439a0-6e5d-44a8-a6b1-e43e09d661b5-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-043439a0-6e5d-44a8-a6b1-e43e09d661b5-0.parquet new file mode 100644 index 000000000..972c59da9 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-043439a0-6e5d-44a8-a6b1-e43e09d661b5-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-10956d53-21db-4d51-83e6-ea39269e80ca-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-10956d53-21db-4d51-83e6-ea39269e80ca-0.parquet new file mode 100644 index 000000000..41d1f780e Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-10956d53-21db-4d51-83e6-ea39269e80ca-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-17bc0233-1c62-46c7-89f1-6093f22e9eee-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-17bc0233-1c62-46c7-89f1-6093f22e9eee-0.parquet new file mode 100644 index 000000000..972c59da9 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-17bc0233-1c62-46c7-89f1-6093f22e9eee-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-1976c24c-7e5d-4179-9bbe-add3159334d0-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-1976c24c-7e5d-4179-9bbe-add3159334d0-0.parquet new file mode 100644 index 000000000..fb1b629e0 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-1976c24c-7e5d-4179-9bbe-add3159334d0-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-9a39d079-dfcb-46f0-8b5b-ef1e35e78711-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-9a39d079-dfcb-46f0-8b5b-ef1e35e78711-0.parquet new file mode 100644 index 000000000..41d1f780e Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-9a39d079-dfcb-46f0-8b5b-ef1e35e78711-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-c59ddaa8-a1c3-4b09-b4a4-9f58a1accd7f-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-c59ddaa8-a1c3-4b09-b4a4-9f58a1accd7f-0.parquet new file mode 100644 index 000000000..53d3e4eaa Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-c59ddaa8-a1c3-4b09-b4a4-9f58a1accd7f-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1141d2e-69d0-4f8f-87f1-110f34c228d7-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1141d2e-69d0-4f8f-87f1-110f34c228d7-0.parquet new file mode 100644 index 000000000..74c4cc132 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1141d2e-69d0-4f8f-87f1-110f34c228d7-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f7ead75f-edd3-4307-9473-f5826baf5d96-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f7ead75f-edd3-4307-9473-f5826baf5d96-0.parquet new file mode 100644 index 000000000..74c4cc132 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f7ead75f-edd3-4307-9473-f5826baf5d96-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f8e3e01a-3620-40bf-9da7-e20669696822-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f8e3e01a-3620-40bf-9da7-e20669696822-0.parquet new file mode 100644 index 000000000..74c4cc132 Binary files /dev/null and b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f8e3e01a-3620-40bf-9da7-e20669696822-0.parquet differ diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 new file mode 100644 index 000000000..ddc738c05 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "pt", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 2, + "name" : "score", + "type" : "INT" + }, { + "id" : 3, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 3, + "partitionKeys" : [ "pt" ], + "primaryKeys" : [ "pt", "id" ], + "options" : { + "bucket" : "2", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "parquet", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866468700 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST new file mode 100644 index 000000000..7813681f5 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 new file mode 100644 index 000000000..2f1c2804b --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "uuid" : "420b22a8-a0e0-4818-adb2-655c492dec96", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-1", + "deltaManifestListSize" : 1119, + "commitUser" : "711e6dbd-2af8-4b27-9793-1cfca17d9c00", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866468750, + "totalRecordCount" : 200, + "deltaRecordCount" : 200, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 new file mode 100644 index 000000000..c4ef63281 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "1f5e235e-3e28-4ab4-812f-6a1e4ca39056", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "b2b00e43-1a8e-4cb1-84da-0f003902e88b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468867, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 new file mode 100644 index 000000000..8d40875fc --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "fa35fcd9-4728-4e75-8e13-d0291a13cb3d", + "id" : 3, + "schemaId" : 0, + "baseManifestList" : "manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-0", + "baseManifestListSize" : 1158, + "deltaManifestList" : "manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "49a0d61e-b7ad-44ac-8a54-d9de191cb80e", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866468901, + "totalRecordCount" : 202, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 new file mode 100644 index 000000000..33f16fd31 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "4e6c3373-056b-4624-8191-1a38f9ba4b23", + "id" : 4, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-0", + "baseManifestListSize" : 1196, + "deltaManifestList" : "manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-a9b54c7b-7468-4a64-97ee-32adae28681e-0", + "commitUser" : "01221812-aecb-4b28-8128-d081617d48a0", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468941, + "totalRecordCount" : 203, + "deltaRecordCount" : 1, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 new file mode 100644 index 000000000..c7926bc1d --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "35a01751-18a9-43d7-8733-4a3ebd1b94c1", + "id" : 5, + "schemaId" : 0, + "baseManifestList" : "manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-0", + "baseManifestListSize" : 1234, + "deltaManifestList" : "manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-1", + "deltaManifestListSize" : 1126, + "indexManifest" : "index-manifest-11be7772-670d-4dc1-adbb-fd8413f2779d-0", + "commitUser" : "8e5448ed-d149-4e78-a7da-10d9616d70b3", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468999, + "totalRecordCount" : 201, + "deltaRecordCount" : -2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base new file mode 100644 index 000000000..c4ef63281 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "1f5e235e-3e28-4ab4-812f-6a1e4ca39056", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "b2b00e43-1a8e-4cb1-84da-0f003902e88b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468867, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file