From 6cadf2a86849b10ee318e2949d75c4c468f48a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=8B=87?= Date: Tue, 11 Aug 2026 02:25:41 -0400 Subject: [PATCH 01/14] feat(core): add primary-key index source metadata and definitions Add the core types for Paimon 2.0 primary-key source-backed scalar indexes: PrimaryKeyIndexSourceMeta v1 decoding (big-endian layout, Java modified UTF-8 file names, defensive count cap, trailing-byte rejection), the COMPACT level>0 source policy, pk-btree / pk-bitmap / pk-vector / pk-full-text definition parsing with field-scoped JSON option validation, and the exact per-level source group validation (PkSortedIndexGroup / PkSortedBucketIndexState) that decides whether a payload covers its data level. The metadata carrier (GlobalIndexMeta source_meta, commit message v12) landed in #179; this change decodes and validates what it carries. Planning and reading follow in the next part. part of #192 --- include/paimon/defs.h | 12 + src/paimon/CMakeLists.txt | 9 + src/paimon/common/defs.cpp | 4 + .../common/utils/java_modified_utf8.cpp | 187 ++++++++++++ src/paimon/common/utils/java_modified_utf8.h | 46 +++ .../common/utils/java_modified_utf8_test.cpp | 116 +++++++ .../index/pk/primary_key_index_definition.h | 72 +++++ .../pk/primary_key_index_definitions.cpp | 232 ++++++++++++++ .../index/pk/primary_key_index_definitions.h | 52 ++++ .../pk/primary_key_index_definitions_test.cpp | 211 +++++++++++++ .../index/pk/primary_key_index_source_file.h | 44 +++ .../pk/primary_key_index_source_meta.cpp | 186 ++++++++++++ .../index/pk/primary_key_index_source_meta.h | 70 +++++ .../pk/primary_key_index_source_meta_test.cpp | 191 ++++++++++++ .../pk/primary_key_index_source_policy.h | 47 +++ .../pksorted/pk_sorted_bucket_index_state.cpp | 101 +++++++ .../pksorted/pk_sorted_bucket_index_state.h | 75 +++++ .../pk_sorted_bucket_index_state_test.cpp | 284 ++++++++++++++++++ .../index/pksorted/pk_sorted_index_group.cpp | 54 ++++ .../index/pksorted/pk_sorted_index_group.h | 77 +++++ 20 files changed, 2070 insertions(+) create mode 100644 src/paimon/common/utils/java_modified_utf8.cpp create mode 100644 src/paimon/common/utils/java_modified_utf8.h create mode 100644 src/paimon/common/utils/java_modified_utf8_test.cpp create mode 100644 src/paimon/core/index/pk/primary_key_index_definition.h create mode 100644 src/paimon/core/index/pk/primary_key_index_definitions.cpp create mode 100644 src/paimon/core/index/pk/primary_key_index_definitions.h create mode 100644 src/paimon/core/index/pk/primary_key_index_definitions_test.cpp create mode 100644 src/paimon/core/index/pk/primary_key_index_source_file.h create mode 100644 src/paimon/core/index/pk/primary_key_index_source_meta.cpp create mode 100644 src/paimon/core/index/pk/primary_key_index_source_meta.h create mode 100644 src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp create mode 100644 src/paimon/core/index/pk/primary_key_index_source_policy.h create mode 100644 src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp create mode 100644 src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h create mode 100644 src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp create mode 100644 src/paimon/core/index/pksorted/pk_sorted_index_group.cpp create mode 100644 src/paimon/core/index/pksorted/pk_sorted_index_group.h diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 46b20c316..90563c4d1 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -519,6 +519,18 @@ struct PAIMON_EXPORT Options { /// "global-index.external-path" - Global index root directory, if not set, the global index /// files will be stored under the index directory. static const char GLOBAL_INDEX_EXTERNAL_PATH[]; + /// "pk-btree.index.columns" - Comma-separated columns indexed by primary-key BTree indexes. + /// No default value. + static const char PK_BTREE_INDEX_COLUMNS[]; + /// "pk-bitmap.index.columns" - Comma-separated columns indexed by primary-key Bitmap indexes. + /// No default value. + static const char PK_BITMAP_INDEX_COLUMNS[]; + /// "pk-vector.index.columns" - Comma-separated VECTOR columns indexed by primary-key vector + /// indexes. No default value. + static const char PK_VECTOR_INDEX_COLUMNS[]; + /// "pk-full-text.index.columns" - Comma-separated character columns indexed by primary-key + /// full-text indexes. No default value. + static const char PK_FULL_TEXT_INDEX_COLUMNS[]; /// "aggregation.remove-record-on-delete" - Whether to remove the whole row in aggregation /// engine when delete records are received. Default value is "false". static const char AGGREGATION_REMOVE_RECORD_ON_DELETE[]; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 3fae8b8e8..4971fa751 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -175,6 +175,7 @@ set(PAIMON_COMMON_SRCS common/data/shredding/shredding_file_reader.cpp common/utils/delta_varint_compressor.cpp common/utils/fields_comparator.cpp + common/utils/java_modified_utf8.cpp common/utils/path_util.cpp common/utils/range.cpp common/utils/read_ahead_cache.cpp @@ -254,6 +255,10 @@ set(PAIMON_CORE_SRCS core/index/index_file_handler.cpp core/index/global_index_meta.cpp core/index/index_file_meta_serializer.cpp + core/index/pk/primary_key_index_source_meta.cpp + core/index/pk/primary_key_index_definitions.cpp + core/index/pksorted/pk_sorted_index_group.cpp + core/index/pksorted/pk_sorted_bucket_index_state.cpp core/io/generic_row_to_arrow_array_converter.cpp core/io/meta_to_arrow_array_converter.cpp core/io/async_key_value_producer_and_consumer.cpp @@ -477,6 +482,7 @@ if(PAIMON_BUILD_TESTS) SOURCES common/memory/memory_pool_test.cpp common/memory/bytes_test.cpp + common/utils/java_modified_utf8_test.cpp common/memory/memory_segment_test.cpp common/memory/memory_segment_utils_test.cpp common/memory/memory_slice_test.cpp @@ -715,6 +721,9 @@ if(PAIMON_BUILD_TESTS) core/index/index_in_data_file_dir_path_factory_test.cpp core/index/deletion_vector_meta_test.cpp core/index/index_file_meta_serializer_test.cpp + core/index/pk/primary_key_index_source_meta_test.cpp + core/index/pk/primary_key_index_definitions_test.cpp + core/index/pksorted/pk_sorted_bucket_index_state_test.cpp core/index/index_file_handler_test.cpp core/io/compact_increment_test.cpp core/io/infer_shredding_file_writer_test.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 53f18c3fd..58d8b4bb6 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -139,6 +139,10 @@ const char Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE[] = "blob-write-null-on-fet const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled"; const char Options::GLOBAL_INDEX_THREAD_NUM[] = "global-index.thread-num"; const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path"; +const char Options::PK_BTREE_INDEX_COLUMNS[] = "pk-btree.index.columns"; +const char Options::PK_BITMAP_INDEX_COLUMNS[] = "pk-bitmap.index.columns"; +const char Options::PK_VECTOR_INDEX_COLUMNS[] = "pk-vector.index.columns"; +const char Options::PK_FULL_TEXT_INDEX_COLUMNS[] = "pk-full-text.index.columns"; const char Options::AGGREGATION_REMOVE_RECORD_ON_DELETE[] = "aggregation.remove-record-on-delete"; const char Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED[] = "table-read.sequence-number.enabled"; const char Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED[] = "key-value.sequence_number.enabled"; diff --git a/src/paimon/common/utils/java_modified_utf8.cpp b/src/paimon/common/utils/java_modified_utf8.cpp new file mode 100644 index 000000000..1303848fa --- /dev/null +++ b/src/paimon/common/utils/java_modified_utf8.cpp @@ -0,0 +1,187 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/java_modified_utf8.h" + +#include + +#include "fmt/format.h" + +namespace paimon { +namespace { +constexpr uint32_t kSupplementaryStart = 0x10000; +constexpr uint32_t kMaxCodePoint = 0x10FFFF; +constexpr uint32_t kHighSurrogateStart = 0xD800; +constexpr uint32_t kLowSurrogateStart = 0xDC00; +constexpr uint32_t kSurrogateEnd = 0xDFFF; + +void AppendTwoBytes(uint32_t code_point, std::string* out) { + out->push_back(static_cast(0xC0 | ((code_point >> 6) & 0x1F))); + out->push_back(static_cast(0x80 | (code_point & 0x3F))); +} + +void AppendThreeBytes(uint32_t code_point, std::string* out) { + out->push_back(static_cast(0xE0 | ((code_point >> 12) & 0x0F))); + out->push_back(static_cast(0x80 | ((code_point >> 6) & 0x3F))); + out->push_back(static_cast(0x80 | (code_point & 0x3F))); +} + +Status MalformedInput(const std::string& what, size_t position) { + return Status::Invalid(fmt::format("Malformed UTF-8 input: {} around byte {}", what, position)); +} +} // namespace + +Result JavaModifiedUtf8::Encode(std::string_view utf8) { + std::string out; + out.reserve(utf8.size()); + size_t i = 0; + while (i < utf8.size()) { + uint8_t byte0 = static_cast(utf8[i]); + if (byte0 < 0x80) { + if (byte0 == 0) { + // Java encodes U+0000 as the overlong two-byte form. + AppendTwoBytes(0, &out); + } else { + out.push_back(static_cast(byte0)); + } + i += 1; + continue; + } + int32_t continuation_count = 0; + uint32_t code_point = 0; + if ((byte0 & 0xE0) == 0xC0) { + continuation_count = 1; + code_point = byte0 & 0x1F; + } else if ((byte0 & 0xF0) == 0xE0) { + continuation_count = 2; + code_point = byte0 & 0x0F; + } else if ((byte0 & 0xF8) == 0xF0) { + continuation_count = 3; + code_point = byte0 & 0x07; + } else { + return MalformedInput("invalid leading byte", i); + } + if (i + continuation_count >= utf8.size()) { + return MalformedInput("truncated sequence", i); + } + for (int32_t k = 1; k <= continuation_count; k++) { + uint8_t continuation = static_cast(utf8[i + k]); + if ((continuation & 0xC0) != 0x80) { + return MalformedInput("invalid continuation byte", i + k); + } + code_point = (code_point << 6) | (continuation & 0x3F); + } + // Reject overlong forms and code points outside Unicode. + static constexpr uint32_t kMinByLength[4] = {0, 0x80, 0x800, kSupplementaryStart}; + if (code_point < kMinByLength[continuation_count] || code_point > kMaxCodePoint || + (code_point >= kHighSurrogateStart && code_point <= kSurrogateEnd)) { + return MalformedInput("invalid code point", i); + } + if (code_point < 0x800) { + AppendTwoBytes(code_point, &out); + } else if (code_point < kSupplementaryStart) { + AppendThreeBytes(code_point, &out); + } else { + // Java writes supplementary code points as a CESU-8 surrogate pair. + uint32_t offset = code_point - kSupplementaryStart; + AppendThreeBytes(kHighSurrogateStart + (offset >> 10), &out); + AppendThreeBytes(kLowSurrogateStart + (offset & 0x3FF), &out); + } + i += 1 + continuation_count; + } + return out; +} + +Result JavaModifiedUtf8::Decode(std::string_view modified_utf8) { + std::string out; + out.reserve(modified_utf8.size()); + size_t i = 0; + // Decoded UTF-16 code units, kept across iterations to pair surrogates. + uint32_t pending_high_surrogate = 0; + bool has_pending_high_surrogate = false; + while (i < modified_utf8.size()) { + uint8_t byte0 = static_cast(modified_utf8[i]); + uint32_t unit = 0; + if (byte0 < 0x80) { + if (byte0 == 0) { + // Java's writeUTF never emits a raw zero byte. + return MalformedInput("unexpected raw zero byte", i); + } + unit = byte0; + i += 1; + } else if ((byte0 & 0xE0) == 0xC0) { + if (i + 1 >= modified_utf8.size()) { + return MalformedInput("truncated two-byte sequence", i); + } + uint8_t byte1 = static_cast(modified_utf8[i + 1]); + if ((byte1 & 0xC0) != 0x80) { + return MalformedInput("invalid continuation byte", i + 1); + } + unit = ((byte0 & 0x1F) << 6) | (byte1 & 0x3F); + i += 2; + } else if ((byte0 & 0xF0) == 0xE0) { + if (i + 2 >= modified_utf8.size()) { + return MalformedInput("truncated three-byte sequence", i); + } + uint8_t byte1 = static_cast(modified_utf8[i + 1]); + uint8_t byte2 = static_cast(modified_utf8[i + 2]); + if ((byte1 & 0xC0) != 0x80 || (byte2 & 0xC0) != 0x80) { + return MalformedInput("invalid continuation byte", i + 1); + } + unit = ((byte0 & 0x0F) << 12) | ((byte1 & 0x3F) << 6) | (byte2 & 0x3F); + i += 3; + } else { + // Java's readUTF rejects four-byte sequences and stray continuation bytes. + return MalformedInput("invalid leading byte", i); + } + + if (has_pending_high_surrogate) { + if (unit >= kLowSurrogateStart && unit <= kSurrogateEnd) { + uint32_t code_point = kSupplementaryStart + + ((pending_high_surrogate - kHighSurrogateStart) << 10) + + (unit - kLowSurrogateStart); + out.push_back(static_cast(0xF0 | ((code_point >> 18) & 0x07))); + out.push_back(static_cast(0x80 | ((code_point >> 12) & 0x3F))); + out.push_back(static_cast(0x80 | ((code_point >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (code_point & 0x3F))); + has_pending_high_surrogate = false; + continue; + } + return MalformedInput("unpaired high surrogate", i); + } + if (unit >= kHighSurrogateStart && unit < kLowSurrogateStart) { + pending_high_surrogate = unit; + has_pending_high_surrogate = true; + continue; + } + if (unit >= kLowSurrogateStart && unit <= kSurrogateEnd) { + return MalformedInput("unpaired low surrogate", i); + } + if (unit < 0x80) { + out.push_back(static_cast(unit)); + } else if (unit < 0x800) { + AppendTwoBytes(unit, &out); + } else { + AppendThreeBytes(unit, &out); + } + } + if (has_pending_high_surrogate) { + return MalformedInput("unpaired high surrogate at end", modified_utf8.size()); + } + return out; +} + +} // namespace paimon diff --git a/src/paimon/common/utils/java_modified_utf8.h b/src/paimon/common/utils/java_modified_utf8.h new file mode 100644 index 000000000..c5fbf5abe --- /dev/null +++ b/src/paimon/common/utils/java_modified_utf8.h @@ -0,0 +1,46 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/result.h" + +namespace paimon { +/// Converts between standard UTF-8 and the "modified UTF-8" used by Java's +/// `DataOutputStream#writeUTF` / `DataInputStream#readUTF`: +/// - U+0000 is encoded as the two-byte sequence 0xC0 0x80 instead of a single zero byte; +/// - supplementary code points (U+10000 and above) are encoded as a UTF-16 surrogate pair, +/// each surrogate written as an independent three-byte sequence (CESU-8), instead of the +/// four-byte standard UTF-8 form. +class JavaModifiedUtf8 { + public: + JavaModifiedUtf8() = delete; + ~JavaModifiedUtf8() = delete; + + /// Encodes a standard UTF-8 string into Java modified UTF-8 bytes. + /// @return An error status if `utf8` is not well-formed UTF-8. + static Result Encode(std::string_view utf8); + + /// Decodes Java modified UTF-8 bytes into a standard UTF-8 string, mirroring the + /// validation of Java's `DataInputStream#readUTF`. + /// @return An error status on any malformed byte sequence. + static Result Decode(std::string_view modified_utf8); +}; + +} // namespace paimon diff --git a/src/paimon/common/utils/java_modified_utf8_test.cpp b/src/paimon/common/utils/java_modified_utf8_test.cpp new file mode 100644 index 000000000..576c2e081 --- /dev/null +++ b/src/paimon/common/utils/java_modified_utf8_test.cpp @@ -0,0 +1,116 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/java_modified_utf8.h" + +#include + +#include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(JavaModifiedUtf8Test, AsciiEncodeIsIdentity) { + std::string ascii = "data-8b2f1a-0.parquet"; + ASSERT_OK_AND_ASSIGN(std::string encoded, JavaModifiedUtf8::Encode(ascii)); + ASSERT_EQ(ascii, encoded); + ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode(encoded)); + ASSERT_EQ(ascii, decoded); +} + +TEST(JavaModifiedUtf8Test, BmpTextRoundTrip) { + // Chinese characters are three-byte sequences, identical in both encodings. + std::string utf8 = "订单表-文件.parquet"; + ASSERT_OK_AND_ASSIGN(std::string encoded, JavaModifiedUtf8::Encode(utf8)); + ASSERT_EQ(utf8, encoded); + ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode(encoded)); + ASSERT_EQ(utf8, decoded); +} + +TEST(JavaModifiedUtf8Test, NulByteUsesOverlongTwoByteForm) { + std::string nul(1, '\0'); + ASSERT_OK_AND_ASSIGN(std::string encoded, JavaModifiedUtf8::Encode(nul)); + ASSERT_EQ("\xC0\x80", encoded); + ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode("\xC0\x80")); + ASSERT_EQ(nul, decoded); + + // U+0000 embedded in surrounding ASCII leaves its neighbors untouched. + std::string embedded("ab\0cd", 5); + ASSERT_OK_AND_ASSIGN(std::string embedded_encoded, JavaModifiedUtf8::Encode(embedded)); + ASSERT_EQ(std::string("ab\xC0\x80" + "cd", + 6), + embedded_encoded); + ASSERT_OK_AND_ASSIGN(std::string embedded_decoded, JavaModifiedUtf8::Decode(embedded_encoded)); + ASSERT_EQ(embedded, embedded_decoded); +} + +TEST(JavaModifiedUtf8Test, SupplementaryCharUsesSurrogatePair) { + // U+1F600 in standard four-byte UTF-8. + std::string standard = "\xF0\x9F\x98\x80"; + ASSERT_OK_AND_ASSIGN(std::string encoded, JavaModifiedUtf8::Encode(standard)); + // CESU-8: surrogate pair U+D83D U+DE00, each written as a three-byte sequence. + ASSERT_EQ("\xED\xA0\xBD\xED\xB8\x80", encoded); + ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode(encoded)); + ASSERT_EQ(standard, decoded); +} + +TEST(JavaModifiedUtf8Test, DecodeRejectsMalformedInput) { + // Java's writeUTF never emits a raw zero byte. + ASSERT_NOK(JavaModifiedUtf8::Decode(std::string(1, '\0'))); + ASSERT_NOK(JavaModifiedUtf8::Decode(std::string("a\0b", 3))); + // Truncated two-byte and three-byte sequences. + ASSERT_NOK(JavaModifiedUtf8::Decode("\xC3")); + ASSERT_NOK(JavaModifiedUtf8::Decode("\xE8\xB8")); + // Continuation bytes must match 10xxxxxx. + ASSERT_NOK(JavaModifiedUtf8::Decode("\xC3\x28")); + ASSERT_NOK(JavaModifiedUtf8::Decode("\xE8\x28\xB8")); + // readUTF rejects four-byte leading bytes; supplementary chars must arrive as CESU-8. + ASSERT_NOK(JavaModifiedUtf8::Decode("\xF0\x9F\x98\x80")); + // Unpaired high surrogate at end of input. + ASSERT_NOK(JavaModifiedUtf8::Decode("\xED\xA0\xBD")); + // High surrogate followed by a non-surrogate unit. + ASSERT_NOK( + JavaModifiedUtf8::Decode("\xED\xA0\xBD" + "z")); + // Low surrogate without a preceding high surrogate. + ASSERT_NOK(JavaModifiedUtf8::Decode("\xED\xB8\x80")); +} + +TEST(JavaModifiedUtf8Test, EncodeRejectsInvalidUtf8) { + // Stray continuation byte. + ASSERT_NOK(JavaModifiedUtf8::Encode("\x80")); + ASSERT_NOK(JavaModifiedUtf8::Encode("a\x80")); + // Truncated multi-byte sequences. + ASSERT_NOK(JavaModifiedUtf8::Encode("\xC3")); + ASSERT_NOK(JavaModifiedUtf8::Encode("\xE8\xB8")); + ASSERT_NOK(JavaModifiedUtf8::Encode("\xF0\x9F\x98")); + // Overlong two-byte encoding of U+002F. + ASSERT_NOK(JavaModifiedUtf8::Encode("\xC0\xAF")); + // Surrogate code point U+D800 encoded directly as a three-byte sequence. + ASSERT_NOK(JavaModifiedUtf8::Encode("\xED\xA0\x80")); + // Code point above U+10FFFF. + ASSERT_NOK(JavaModifiedUtf8::Encode("\xF4\x90\x80\x80")); +} + +TEST(JavaModifiedUtf8Test, DecodeAcceptsJavaLenientOverlongTwoByteForm) { + // Java's readUTF only pattern-matches the bit layout of two-byte sequences, so the + // overlong encoding 0xC1 0xBF of U+007F is accepted; Decode mirrors that leniency. + ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode("\xC1\xBF")); + ASSERT_EQ("\x7F", decoded); +} + +} // namespace paimon::test diff --git a/src/paimon/core/index/pk/primary_key_index_definition.h b/src/paimon/core/index/pk/primary_key_index_definition.h new file mode 100644 index 000000000..3506877e8 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_definition.h @@ -0,0 +1,72 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace paimon { +/// Resolved definition of one source-backed primary-key index. +class PrimaryKeyIndexDefinition { + public: + /// Built-in primary-key index families. + enum class Family { + VECTOR, + BTREE, + BITMAP, + FULL_TEXT, + }; + + PrimaryKeyIndexDefinition(std::string column, int32_t field_id, std::string index_type, + std::map options, Family family) + : column_(std::move(column)), + field_id_(field_id), + index_type_(std::move(index_type)), + options_(std::move(options)), + family_(family) {} + + const std::string& Column() const { + return column_; + } + + int32_t FieldId() const { + return field_id_; + } + + const std::string& IndexType() const { + return index_type_; + } + + const std::map& Options() const { + return options_; + } + + Family GetFamily() const { + return family_; + } + + private: + std::string column_; + int32_t field_id_; + std::string index_type_; + std::map options_; + Family family_; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.cpp b/src/paimon/core/index/pk/primary_key_index_definitions.cpp new file mode 100644 index 000000000..e23a51c62 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_definitions.cpp @@ -0,0 +1,232 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/index/pk/primary_key_index_definitions.h" + +#include +#include + +#include "fmt/format.h" +#include "paimon/defs.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace paimon { +namespace { +constexpr char kBTreeIndexType[] = "btree"; +constexpr char kBitmapIndexType[] = "bitmap"; +constexpr char kFullTextIndexType[] = "full-text"; +constexpr char kBTreeOptionFamily[] = "pk-btree"; +constexpr char kBitmapOptionFamily[] = "pk-bitmap"; +constexpr char kBTreeAlgorithmPrefix[] = "btree-index."; +constexpr char kBitmapAlgorithmPrefix[] = "bitmap-index."; +constexpr char kFieldScopedPrefix[] = "fields."; +constexpr char kRecordsPerRangeKey[] = "sorted-index.records-per-range"; + +std::string Trim(const std::string& value) { + size_t begin = value.find_first_not_of(" \t\r\n"); + if (begin == std::string::npos) { + return ""; + } + size_t end = value.find_last_not_of(" \t\r\n"); + return value.substr(begin, end - begin + 1); +} + +std::vector IndexColumns(const std::map& options, + const char* option_key) { + auto iter = options.find(option_key); + if (iter == options.end()) { + return {}; + } + std::vector columns; + const std::string& value = iter->second; + size_t start = 0; + while (true) { + size_t comma = value.find(',', start); + if (comma == std::string::npos) { + columns.push_back(Trim(value.substr(start))); + break; + } + columns.push_back(Trim(value.substr(start, comma - start))); + start = comma + 1; + } + return columns; +} + +Status ValidateNoDuplicates(const std::vector& columns, const char* option_key) { + std::set unique_columns; + for (const std::string& column : columns) { + if (!unique_columns.insert(column).second) { + return Status::Invalid( + fmt::format("{} contains duplicate column '{}'.", option_key, column)); + } + } + return Status::OK(); +} + +Status ValidateUniqueColumns(std::set* indexed_columns, + const std::vector& columns) { + for (const std::string& column : columns) { + if (!indexed_columns->insert(column).second) { + return Status::Invalid( + fmt::format("Column '{}' can own at most one primary-key index.", column)); + } + } + return Status::OK(); +} + +bool StartsWith(const std::string& value, const char* prefix) { + return value.rfind(prefix, 0) == 0; +} + +/// Resolves the effective option map of one sorted-index definition: table options first, +/// then the field-scoped JSON options with unqualified keys prefixed by the algorithm +/// prefix, mirroring Java `CoreOptions#primaryKeySortedIndexOptions`. +Result> SortedIndexOptions( + const std::map& table_options, const std::string& column, + const char* option_family, const char* algorithm_prefix) { + std::map resolved = table_options; + resolved.erase(kRecordsPerRangeKey); + std::string option_key = + fmt::format("{}{}.{}.index.options", kFieldScopedPrefix, column, option_family); + auto iter = table_options.find(option_key); + if (iter == table_options.end() || Trim(iter->second).empty()) { + return resolved; + } + + rapidjson::Document document; + document.Parse(iter->second.c_str()); + if (document.HasParseError() || !document.IsObject()) { + return Status::Invalid( + fmt::format("{} must be a JSON object of option key-value pairs.", option_key)); + } + for (auto member = document.MemberBegin(); member != document.MemberEnd(); ++member) { + if (!member->name.IsString() || Trim(member->name.GetString()).empty()) { + return Status::Invalid(fmt::format("{} contains an empty option key.", option_key)); + } + std::string key = member->name.GetString(); + if (member->value.IsNull()) { + return Status::Invalid(fmt::format("{} value for key {} must not be null.", + option_key, key)); + } + if (member->value.IsObject() || member->value.IsArray()) { + return Status::Invalid( + fmt::format("{} must be a JSON object of option key-value pairs.", option_key)); + } + std::string value; + if (member->value.IsString()) { + value = member->value.GetString(); + } else { + // Java's parseJsonMap(..., String.class) coerces scalar JSON values (numbers, + // booleans) to their text form, so `{"compression-level":3}` is valid there. + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + member->value.Accept(writer); + value = buffer.GetString(); + } + std::string qualified_key = + StartsWith(key, algorithm_prefix) || StartsWith(key, kFieldScopedPrefix) + ? key + : algorithm_prefix + key; + auto previous = resolved.find(qualified_key); + if (previous != resolved.end() && previous->second != value) { + return Status::Invalid( + fmt::format("{} defines conflicting values for {}.", option_key, qualified_key)); + } + resolved[qualified_key] = value; + } + return resolved; +} + +bool Contains(const std::vector& columns, const std::string& column) { + for (const std::string& candidate : columns) { + if (candidate == column) { + return true; + } + } + return false; +} +} // namespace + +Result PrimaryKeyIndexDefinitions::Create(const TableSchema& schema) { + const std::map& options = schema.Options(); + std::vector vector_columns = + IndexColumns(options, Options::PK_VECTOR_INDEX_COLUMNS); + std::vector btree_columns = IndexColumns(options, Options::PK_BTREE_INDEX_COLUMNS); + std::vector bitmap_columns = + IndexColumns(options, Options::PK_BITMAP_INDEX_COLUMNS); + std::vector full_text_columns = + IndexColumns(options, Options::PK_FULL_TEXT_INDEX_COLUMNS); + PAIMON_RETURN_NOT_OK(ValidateNoDuplicates(vector_columns, Options::PK_VECTOR_INDEX_COLUMNS)); + PAIMON_RETURN_NOT_OK(ValidateNoDuplicates(btree_columns, Options::PK_BTREE_INDEX_COLUMNS)); + PAIMON_RETURN_NOT_OK(ValidateNoDuplicates(bitmap_columns, Options::PK_BITMAP_INDEX_COLUMNS)); + PAIMON_RETURN_NOT_OK( + ValidateNoDuplicates(full_text_columns, Options::PK_FULL_TEXT_INDEX_COLUMNS)); + std::set indexed_columns; + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(&indexed_columns, vector_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(&indexed_columns, btree_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(&indexed_columns, bitmap_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(&indexed_columns, full_text_columns)); + + std::vector definitions; + for (const DataField& field : schema.Fields()) { + const std::string& column = field.Name(); + if (Contains(btree_columns, column)) { + Result> definition_options = + SortedIndexOptions(options, column, kBTreeOptionFamily, kBTreeAlgorithmPrefix); + PAIMON_RETURN_NOT_OK(definition_options.status()); + definitions.emplace_back(column, field.Id(), kBTreeIndexType, + std::move(definition_options).value(), + PrimaryKeyIndexDefinition::Family::BTREE); + } else if (Contains(bitmap_columns, column)) { + Result> definition_options = + SortedIndexOptions(options, column, kBitmapOptionFamily, kBitmapAlgorithmPrefix); + PAIMON_RETURN_NOT_OK(definition_options.status()); + definitions.emplace_back(column, field.Id(), kBitmapIndexType, + std::move(definition_options).value(), + PrimaryKeyIndexDefinition::Family::BITMAP); + } else if (Contains(vector_columns, column)) { + std::string index_type; + auto type_iter = + options.find(fmt::format("{}{}.pk-vector.index.type", kFieldScopedPrefix, column)); + if (type_iter != options.end()) { + index_type = type_iter->second; + } + definitions.emplace_back(column, field.Id(), index_type, + std::map(), + PrimaryKeyIndexDefinition::Family::VECTOR); + } else if (Contains(full_text_columns, column)) { + definitions.emplace_back(column, field.Id(), kFullTextIndexType, + std::map(), + PrimaryKeyIndexDefinition::Family::FULL_TEXT); + } + } + return PrimaryKeyIndexDefinitions(std::move(definitions)); +} + +std::vector PrimaryKeyIndexDefinitions::ScalarDefinitions() const { + std::vector scalar_definitions; + for (const PrimaryKeyIndexDefinition& definition : definitions_) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE || + definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BITMAP) { + scalar_definitions.push_back(definition); + } + } + return scalar_definitions; +} + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.h b/src/paimon/core/index/pk/primary_key_index_definitions.h new file mode 100644 index 000000000..c85fdd629 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_definitions.h @@ -0,0 +1,52 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/index/pk/primary_key_index_definition.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/result.h" + +namespace paimon { +/// Resolves all configured source-backed primary-key indexes of a table schema. +class PrimaryKeyIndexDefinitions { + public: + /// Resolves index definitions from `pk-btree.index.columns`, `pk-bitmap.index.columns`, + /// `pk-vector.index.columns` and `pk-full-text.index.columns` together with their + /// field-scoped option JSON, rejecting duplicate columns and columns owned by more than + /// one index family. + static Result Create(const TableSchema& schema); + + const std::vector& Definitions() const { + return definitions_; + } + + /// @return The scalar (BTree / Bitmap) definitions usable by batch scans. + std::vector ScalarDefinitions() const; + + private: + explicit PrimaryKeyIndexDefinitions(std::vector definitions) + : definitions_(std::move(definitions)) {} + + std::vector definitions_; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp b/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp new file mode 100644 index 000000000..86595b0e8 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp @@ -0,0 +1,211 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/index/pk/primary_key_index_definitions.h" + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/index/pk/primary_key_index_definition.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/defs.h" +#include "paimon/result.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { +/// Builds a primary-key table schema with fields id BIGINT (pk), price DOUBLE, age INT, +/// status STRING and emb FLOAT, merging the given options over a fixed bucket option. +Result> MakeSchema(std::map options) { + std::vector fields = { + DataField(0, arrow::field("id", arrow::int64(), /*nullable=*/false)), + DataField(1, arrow::field("price", arrow::float64())), + DataField(2, arrow::field("age", arrow::int32())), + DataField(3, arrow::field("status", arrow::utf8())), + DataField(4, arrow::field("emb", arrow::float32()))}; + options.emplace(Options::BUCKET, "1"); + return TableSchema::Create(/*schema_id=*/0, DataField::ConvertDataFieldsToArrowSchema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options); +} +} // namespace + +TEST(PrimaryKeyIndexDefinitionsTest, NoIndexOptionsYieldsEmptyDefinitions) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, MakeSchema({})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_TRUE(definitions.Definitions().empty()); + ASSERT_TRUE(definitions.ScalarDefinitions().empty()); +} + +TEST(PrimaryKeyIndexDefinitionsTest, ResolvesBTreeDefinitions) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price,age"}})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_EQ(2, definitions.Definitions().size()); + const PrimaryKeyIndexDefinition& price = definitions.Definitions()[0]; + ASSERT_EQ("price", price.Column()); + ASSERT_EQ(1, price.FieldId()); + ASSERT_EQ("btree", price.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::BTREE, price.GetFamily()); + const PrimaryKeyIndexDefinition& age = definitions.Definitions()[1]; + ASSERT_EQ("age", age.Column()); + ASSERT_EQ(2, age.FieldId()); + ASSERT_EQ("btree", age.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::BTREE, age.GetFamily()); + ASSERT_EQ(2, definitions.ScalarDefinitions().size()); +} + +TEST(PrimaryKeyIndexDefinitionsTest, ResolvesBitmapDefinition) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BITMAP_INDEX_COLUMNS, "status"}})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_EQ(1, definitions.Definitions().size()); + const PrimaryKeyIndexDefinition& status = definitions.Definitions()[0]; + ASSERT_EQ("status", status.Column()); + ASSERT_EQ(3, status.FieldId()); + ASSERT_EQ("bitmap", status.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::BITMAP, status.GetFamily()); + ASSERT_EQ(1, definitions.ScalarDefinitions().size()); +} + +TEST(PrimaryKeyIndexDefinitionsTest, IgnoresColumnAbsentFromSchema) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "not_in_schema"}})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_TRUE(definitions.Definitions().empty()); +} + +TEST(PrimaryKeyIndexDefinitionsTest, CoercesScalarJsonOptionValuesLikeJava) { + // Java's parseJsonMap(..., String.class) accepts scalar JSON values and coerces them + // to text, so numeric or boolean values written by a Java engine must stay readable. + std::map options = { + {Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", + R"({"compression-level":3,"cache-enabled":true,"block-size":"64 kb"})"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, MakeSchema(options)); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + const std::map& resolved = definitions.Definitions()[0].Options(); + ASSERT_EQ("3", resolved.at("btree-index.compression-level")); + ASSERT_EQ("true", resolved.at("btree-index.cache-enabled")); + ASSERT_EQ("64 kb", resolved.at("btree-index.block-size")); + + // Null and nested values are rejected like in Java. + std::map null_options = { + {Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", R"({"block-size":null})"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr null_schema, MakeSchema(null_options)); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*null_schema)); + std::map nested_options = { + {Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", R"({"block-size":{"v":"64 kb"}})"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr nested_schema, MakeSchema(nested_options)); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*nested_schema)); +} + +TEST(PrimaryKeyIndexDefinitionsTest, QualifiesFieldScopedJsonOptions) { + std::map options = { + {Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"sorted-index.records-per-range", "4096"}, + {"fields.price.pk-btree.index.options", + R"({"block-size":"64 kb","btree-index.cache-size":"32 mb","fields.foo.x":"y"})"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, MakeSchema(options)); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_EQ(1, definitions.Definitions().size()); + const std::map& resolved = definitions.Definitions()[0].Options(); + // Unqualified keys are prefixed with the algorithm prefix, qualified keys are kept as-is. + ASSERT_EQ(1, resolved.count("btree-index.block-size")); + ASSERT_EQ("64 kb", resolved.at("btree-index.block-size")); + ASSERT_EQ(1, resolved.count("btree-index.cache-size")); + ASSERT_EQ("32 mb", resolved.at("btree-index.cache-size")); + ASSERT_EQ(1, resolved.count("fields.foo.x")); + ASSERT_EQ("y", resolved.at("fields.foo.x")); + // The per-range knob never leaks into the definition, other table options are retained. + ASSERT_EQ(0, resolved.count("sorted-index.records-per-range")); + ASSERT_EQ(1, resolved.count(Options::BUCKET)); + ASSERT_EQ("1", resolved.at(Options::BUCKET)); +} + +TEST(PrimaryKeyIndexDefinitionsTest, RejectsConflictingJsonOptionValue) { + ASSERT_OK_AND_ASSIGN( + std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"btree-index.block-size", "128 kb"}, + {"fields.price.pk-btree.index.options", R"({"block-size":"64 kb"})"}})); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); +} + +TEST(PrimaryKeyIndexDefinitionsTest, RejectsMalformedJsonOptions) { + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", "not-json"}})); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); + } + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {"fields.price.pk-btree.index.options", R"({"":"v"})"}})); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); + } +} + +TEST(PrimaryKeyIndexDefinitionsTest, RejectsDuplicateColumnWithinFamily) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price,price"}})); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); +} + +TEST(PrimaryKeyIndexDefinitionsTest, RejectsColumnSharedAcrossFamilies) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {Options::PK_BITMAP_INDEX_COLUMNS, "price"}})); + ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); +} + +TEST(PrimaryKeyIndexDefinitionsTest, ResolvesNonScalarFamiliesAndExcludesThemFromScalar) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, + MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, + {Options::PK_VECTOR_INDEX_COLUMNS, "emb"}, + {"fields.emb.pk-vector.index.type", "ivf-flat"}, + {Options::PK_FULL_TEXT_INDEX_COLUMNS, "status"}})); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_EQ(3, definitions.Definitions().size()); + const PrimaryKeyIndexDefinition& full_text = definitions.Definitions()[1]; + ASSERT_EQ("status", full_text.Column()); + ASSERT_EQ("full-text", full_text.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::FULL_TEXT, full_text.GetFamily()); + const PrimaryKeyIndexDefinition& embedding = definitions.Definitions()[2]; + ASSERT_EQ("emb", embedding.Column()); + ASSERT_EQ(4, embedding.FieldId()); + ASSERT_EQ("ivf-flat", embedding.IndexType()); + ASSERT_EQ(PrimaryKeyIndexDefinition::Family::VECTOR, embedding.GetFamily()); + std::vector scalar_definitions = definitions.ScalarDefinitions(); + ASSERT_EQ(1, scalar_definitions.size()); + ASSERT_EQ("price", scalar_definitions[0].Column()); +} + +} // namespace paimon::test diff --git a/src/paimon/core/index/pk/primary_key_index_source_file.h b/src/paimon/core/index/pk/primary_key_index_source_file.h new file mode 100644 index 000000000..23d9c303c --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_file.h @@ -0,0 +1,44 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +namespace paimon { +/// One ordered source data file covered by a source-backed primary-key index payload. +/// +/// Two source files are interchangeable only when both the file name and the row count +/// match; coverage validation relies on this strict identity. +struct PrimaryKeyIndexSourceFile { + PrimaryKeyIndexSourceFile(std::string _file_name, int64_t _row_count) + : file_name(std::move(_file_name)), row_count(_row_count) {} + + bool operator==(const PrimaryKeyIndexSourceFile& other) const { + return file_name == other.file_name && row_count == other.row_count; + } + + bool operator!=(const PrimaryKeyIndexSourceFile& other) const { + return !(*this == other); + } + + std::string file_name; + int64_t row_count; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp new file mode 100644 index 000000000..646b50881 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp @@ -0,0 +1,186 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/index/pk/primary_key_index_source_meta.h" + +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/java_modified_utf8.h" +#include "paimon/core/index/index_file_meta.h" + +namespace paimon { +namespace { +// Each serialized entry needs at least the two-byte writeUTF length and one int64 row count, +// mirroring the defensive source file count cap of the Java deserializer. +constexpr size_t kMinBytesPerSourceFile = sizeof(uint16_t) + sizeof(int64_t); + +void AppendBigEndian32(int32_t value, std::string* out) { + uint32_t bits = static_cast(value); + out->push_back(static_cast((bits >> 24) & 0xFF)); + out->push_back(static_cast((bits >> 16) & 0xFF)); + out->push_back(static_cast((bits >> 8) & 0xFF)); + out->push_back(static_cast(bits & 0xFF)); +} + +void AppendBigEndian64(int64_t value, std::string* out) { + uint64_t bits = static_cast(value); + for (int32_t shift = 56; shift >= 0; shift -= 8) { + out->push_back(static_cast((bits >> shift) & 0xFF)); + } +} + +class BigEndianCursor { + public: + BigEndianCursor(const char* data, size_t length) : data_(data), length_(length) {} + + Result ReadInt32() { + PAIMON_RETURN_NOT_OK(CheckAvailable(sizeof(int32_t))); + uint32_t bits = 0; + for (size_t k = 0; k < sizeof(int32_t); k++) { + bits = (bits << 8) | static_cast(data_[position_ + k]); + } + position_ += sizeof(int32_t); + return static_cast(bits); + } + + Result ReadInt64() { + PAIMON_RETURN_NOT_OK(CheckAvailable(sizeof(int64_t))); + uint64_t bits = 0; + for (size_t k = 0; k < sizeof(int64_t); k++) { + bits = (bits << 8) | static_cast(data_[position_ + k]); + } + position_ += sizeof(int64_t); + return static_cast(bits); + } + + Result ReadUint16() { + PAIMON_RETURN_NOT_OK(CheckAvailable(sizeof(uint16_t))); + uint16_t bits = static_cast((static_cast(data_[position_]) << 8) | + static_cast(data_[position_ + 1])); + position_ += sizeof(uint16_t); + return bits; + } + + Result ReadBytes(size_t length) { + PAIMON_RETURN_NOT_OK(CheckAvailable(length)); + std::string_view view(data_ + position_, length); + position_ += length; + return view; + } + + size_t Available() const { + return length_ - position_; + } + + private: + Status CheckAvailable(size_t needed) const { + if (length_ - position_ < needed) { + return Status::Invalid(fmt::format( + "Failed to deserialize index source metadata: need {} bytes at offset {} but " + "only {} remain.", + needed, position_, length_ - position_)); + } + return Status::OK(); + } + + const char* data_; + size_t length_; + size_t position_ = 0; +}; +} // namespace + +Result PrimaryKeyIndexSourceMeta::Create( + int32_t data_level, std::vector source_files) { + if (data_level <= 0) { + return Status::Invalid("Primary-key index data level must be positive."); + } + if (source_files.empty()) { + return Status::Invalid("An index must reference source files."); + } + return PrimaryKeyIndexSourceMeta(data_level, std::move(source_files)); +} + +Result PrimaryKeyIndexSourceMeta::FromIndexFile( + const IndexFileMeta& index_file) { + const std::optional& global_index_meta = index_file.GetGlobalIndexMeta(); + if (global_index_meta == std::nullopt || global_index_meta.value().source_meta == nullptr) { + return Status::Invalid( + fmt::format("Index file {} has no source metadata.", index_file.FileName())); + } + const std::shared_ptr& source_meta = global_index_meta.value().source_meta; + return Deserialize(source_meta->data(), source_meta->size()); +} + +Result PrimaryKeyIndexSourceMeta::Deserialize(const char* data, + size_t length) { + BigEndianCursor cursor(data, length); + PAIMON_ASSIGN_OR_RAISE(int32_t version, cursor.ReadInt32()); + if (version != VERSION) { + return Status::Invalid(fmt::format("Unsupported index source version: {}.", version)); + } + PAIMON_ASSIGN_OR_RAISE(int32_t data_level, cursor.ReadInt32()); + PAIMON_ASSIGN_OR_RAISE(int32_t source_file_count, cursor.ReadInt32()); + if (source_file_count <= 0) { + return Status::Invalid("An index must reference source files."); + } + size_t maximum_source_file_count = cursor.Available() / kMinBytesPerSourceFile; + if (static_cast(source_file_count) > maximum_source_file_count) { + return Status::Invalid(fmt::format( + "Failed to deserialize index source metadata: source file count {} exceeds the " + "maximum {} allowed by the remaining bytes.", + source_file_count, maximum_source_file_count)); + } + std::vector source_files; + source_files.reserve(source_file_count); + for (int32_t i = 0; i < source_file_count; i++) { + PAIMON_ASSIGN_OR_RAISE(uint16_t name_length, cursor.ReadUint16()); + PAIMON_ASSIGN_OR_RAISE(std::string_view name_bytes, cursor.ReadBytes(name_length)); + PAIMON_ASSIGN_OR_RAISE(std::string file_name, JavaModifiedUtf8::Decode(name_bytes)); + PAIMON_ASSIGN_OR_RAISE(int64_t row_count, cursor.ReadInt64()); + source_files.emplace_back(std::move(file_name), row_count); + } + if (cursor.Available() != 0) { + return Status::Invalid("Unexpected trailing bytes in index source metadata."); + } + return Create(data_level, std::move(source_files)); +} + +Result> PrimaryKeyIndexSourceMeta::Serialize(MemoryPool* pool) const { + std::string buffer; + AppendBigEndian32(VERSION, &buffer); + AppendBigEndian32(data_level_, &buffer); + AppendBigEndian32(static_cast(source_files_.size()), &buffer); + for (const PrimaryKeyIndexSourceFile& source_file : source_files_) { + PAIMON_ASSIGN_OR_RAISE(std::string encoded_name, + JavaModifiedUtf8::Encode(source_file.file_name)); + if (encoded_name.size() > std::numeric_limits::max()) { + return Status::Invalid(fmt::format( + "Source file name is too long for writeUTF: {} bytes.", encoded_name.size())); + } + uint16_t name_length = static_cast(encoded_name.size()); + buffer.push_back(static_cast((name_length >> 8) & 0xFF)); + buffer.push_back(static_cast(name_length & 0xFF)); + buffer.append(encoded_name); + AppendBigEndian64(source_file.row_count, &buffer); + } + return std::make_shared(buffer, pool); +} + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.h b/src/paimon/core/index/pk/primary_key_index_source_meta.h new file mode 100644 index 000000000..97ce35407 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.h @@ -0,0 +1,70 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { +class IndexFileMeta; + +/// Ordered source data files covered by a source-backed primary-key index payload. +/// +/// Wire format (version 1), byte-compatible with Java `PrimaryKeyIndexSourceMeta`: +/// big-endian int32 version, big-endian int32 data level (> 0), big-endian int32 source +/// file count (> 0), then per source file a Java `writeUTF` file name (uint16 big-endian +/// byte length + modified UTF-8 bytes) and a big-endian int64 row count. Trailing bytes +/// are rejected. +class PrimaryKeyIndexSourceMeta { + public: + static constexpr int32_t VERSION = 1; + + static Result Create( + int32_t data_level, std::vector source_files); + + /// Extracts and deserializes the source metadata carried by an index file. + static Result FromIndexFile(const IndexFileMeta& index_file); + + static Result Deserialize(const char* data, size_t length); + + Result> Serialize(MemoryPool* pool) const; + + int32_t DataLevel() const { + return data_level_; + } + + const std::vector& SourceFiles() const { + return source_files_; + } + + private: + PrimaryKeyIndexSourceMeta(int32_t data_level, + std::vector source_files) + : data_level_(data_level), source_files_(std::move(source_files)) {} + + int32_t data_level_; + std::vector source_files_; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp new file mode 100644 index 000000000..2311d6861 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp @@ -0,0 +1,191 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/index/pk/primary_key_index_source_meta.h" + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class PrimaryKeyIndexSourceMetaTest : public ::testing::Test { + protected: + void SetUp() override { + pool_ = GetDefaultPool(); + } + + Result SerializeToString(int32_t data_level, + std::vector source_files) { + PAIMON_ASSIGN_OR_RAISE( + PrimaryKeyIndexSourceMeta meta, + PrimaryKeyIndexSourceMeta::Create(data_level, std::move(source_files))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, meta.Serialize(pool_.get())); + return std::string(bytes->data(), bytes->size()); + } + + std::shared_ptr pool_; +}; + +TEST_F(PrimaryKeyIndexSourceMetaTest, SerializeMatchesGoldenBytes) { + std::vector files; + files.emplace_back("a.parquet", 100); + files.emplace_back("b.parquet", 200); + ASSERT_OK_AND_ASSIGN(std::string serialized, SerializeToString(3, files)); + + const uint8_t kExpected[] = { + 0x00, 0x00, 0x00, 0x01, // version 1 + 0x00, 0x00, 0x00, 0x03, // data level 3 + 0x00, 0x00, 0x00, 0x02, // source file count 2 + 0x00, 0x09, // writeUTF byte length of "a.parquet" + 'a', '.', 'p', 'a', 'r', 'q', 'u', 'e', 't', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, // row count 100 + 0x00, 0x09, // writeUTF byte length of "b.parquet" + 'b', '.', 'p', 'a', 'r', 'q', 'u', 'e', 't', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC8, // row count 200 + }; + std::string expected(reinterpret_cast(kExpected), sizeof(kExpected)); + ASSERT_EQ(expected, serialized); + + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta meta, PrimaryKeyIndexSourceMeta::Deserialize( + serialized.data(), serialized.size())); + ASSERT_EQ(3, meta.DataLevel()); + ASSERT_EQ(files, meta.SourceFiles()); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, RoundTripWithChineseNameAndLargeRowCount) { + std::vector files; + // Row count above 2^32 exercises the full big-endian int64 encoding. + files.emplace_back("文件-0.parquet", (int64_t{1} << 40) + 7); + files.emplace_back("data-1.parquet", 42); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta meta, + PrimaryKeyIndexSourceMeta::Create(5, files)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, meta.Serialize(pool_.get())); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta decoded, + PrimaryKeyIndexSourceMeta::Deserialize(bytes->data(), bytes->size())); + ASSERT_EQ(5, decoded.DataLevel()); + ASSERT_EQ(files, decoded.SourceFiles()); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, DeserializeRejectsBadHeaders) { + std::vector files; + files.emplace_back("a.parquet", 100); + ASSERT_OK_AND_ASSIGN(std::string valid, SerializeToString(3, files)); + // Layout: [0,4) version, [4,8) data level, [8,12) count, [12,14) name length, + // [14,23) name bytes, [23,31) row count. + ASSERT_EQ(static_cast(31), valid.size()); + + // Unsupported versions. + std::string version_two = valid; + version_two[3] = '\x02'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(version_two.data(), version_two.size())); + std::string version_zero = valid; + version_zero[3] = '\x00'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(version_zero.data(), version_zero.size())); + + // Source file count must be positive. + std::string zero_count = valid; + zero_count[11] = '\x00'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(zero_count.data(), zero_count.size())); + std::string negative_count = valid; + for (size_t i = 8; i < 12; i++) { + negative_count[i] = '\xFF'; + } + ASSERT_NOK( + PrimaryKeyIndexSourceMeta::Deserialize(negative_count.data(), negative_count.size())); + + // Claimed count 1000 exceeds the defensive cap allowed by the 19 remaining bytes. + std::string huge_count = valid; + huge_count[10] = '\x03'; + huge_count[11] = '\xE8'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(huge_count.data(), huge_count.size())); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, DeserializeRejectsBadPayloads) { + std::vector files; + files.emplace_back("a.parquet", 100); + ASSERT_OK_AND_ASSIGN(std::string valid, SerializeToString(3, files)); + ASSERT_EQ(static_cast(31), valid.size()); + + // Trailing bytes after a valid payload. + std::string trailing = valid + '\x00'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(trailing.data(), trailing.size())); + + // Buffer cut in the middle of the file name: only 8 of the 9 name bytes remain. + std::string cut_name = valid.substr(0, 22); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(cut_name.data(), cut_name.size())); + + // Buffer cut in the middle of the row count: only 4 of the 8 bytes remain. + std::string cut_row_count = valid.substr(0, 27); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(cut_row_count.data(), cut_row_count.size())); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, CreateRejectsInvalidArguments) { + std::vector files; + files.emplace_back("a.parquet", 100); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(0, files)); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(-1, files)); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(3, {})); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, FromIndexFileDecodesSourceMeta) { + std::vector files; + files.emplace_back("a.parquet", 100); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta meta, + PrimaryKeyIndexSourceMeta::Create(7, files)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr source_meta, meta.Serialize(pool_.get())); + std::shared_ptr index_meta = std::make_shared("index-payload", pool_.get()); + GlobalIndexMeta global_index_meta(/*_row_range_start=*/0, /*_row_range_end=*/100, + /*_index_field_id=*/1, /*_extra_field_ids=*/std::nullopt, + index_meta, source_meta); + IndexFileMeta index_file("pk-btree", "index-file-0", /*file_size=*/64, /*row_count=*/100, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, + global_index_meta); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta decoded, + PrimaryKeyIndexSourceMeta::FromIndexFile(index_file)); + ASSERT_EQ(7, decoded.DataLevel()); + ASSERT_EQ(files, decoded.SourceFiles()); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, FromIndexFileRejectsMissingSourceMeta) { + // Index file without any global index metadata. + IndexFileMeta no_global_index("pk-btree", "index-file-1", /*file_size=*/64, /*row_count=*/100, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::FromIndexFile(no_global_index)); + + // Global index metadata whose source_meta is null. + std::shared_ptr index_meta = std::make_shared("index-payload", pool_.get()); + GlobalIndexMeta null_source_meta(/*_row_range_start=*/0, /*_row_range_end=*/100, + /*_index_field_id=*/1, /*_extra_field_ids=*/std::nullopt, + index_meta, /*_source_meta=*/nullptr); + IndexFileMeta no_source_meta("pk-btree", "index-file-2", /*file_size=*/64, /*row_count=*/100, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, + null_source_meta); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::FromIndexFile(no_source_meta)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/index/pk/primary_key_index_source_policy.h b/src/paimon/core/index/pk/primary_key_index_source_policy.h new file mode 100644 index 000000000..b6c366461 --- /dev/null +++ b/src/paimon/core/index/pk/primary_key_index_source_policy.h @@ -0,0 +1,47 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_source.h" + +namespace paimon { +/// Selects complete compacted data files for source-backed primary-key indexes. +/// +/// Only files produced by compaction on a positive data level are eligible: level 0 files +/// and appended files may still be rewritten or merged, so payloads built over them could +/// not maintain the exact per-level coverage contract. +class PrimaryKeyIndexSourcePolicy { + public: + PrimaryKeyIndexSourcePolicy() = delete; + ~PrimaryKeyIndexSourcePolicy() = delete; + + static bool ShouldWrite(const FileSource& file_source, int32_t level) { + return file_source == FileSource::Compact() && level > 0; + } + + static bool ShouldRead(const DataFileMeta& file) { + if (file.file_source == std::nullopt) { + return false; + } + return ShouldWrite(file.file_source.value(), file.level); + } +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp new file mode 100644 index 000000000..e86bf0956 --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp @@ -0,0 +1,101 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/index/pksorted/pk_sorted_bucket_index_state.h" + +#include +#include +#include +#include + +#include "paimon/core/index/pk/primary_key_index_source_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_policy.h" + +namespace paimon { +PkSortedBucketIndexState PkSortedBucketIndexState::FromActiveDataFiles( + int32_t field_id, const std::string& index_type, + const std::vector>& active_data_files, + const std::vector>& active_payloads) { + std::map> sources_by_level; + for (const std::shared_ptr& data_file : active_data_files) { + if (data_file != nullptr && PrimaryKeyIndexSourcePolicy::ShouldRead(*data_file)) { + sources_by_level[data_file->level].emplace_back(data_file->file_name, + data_file->row_count); + } + } + for (auto& level_sources : sources_by_level) { + std::sort( + level_sources.second.begin(), level_sources.second.end(), + [](const PrimaryKeyIndexSourceFile& left, const PrimaryKeyIndexSourceFile& right) { + return left.file_name < right.file_name; + }); + } + + // Match payloads against the expected level sources; anything that does not decode or + // does not exactly cover its level is rejected. + std::map>> payloads_by_level; + std::map> payload_metas_by_level; + std::vector> rejected; + for (const std::shared_ptr& payload : active_payloads) { + if (payload == nullptr) { + continue; + } + Result source_meta_result = + PrimaryKeyIndexSourceMeta::FromIndexFile(*payload); + if (!source_meta_result.ok()) { + rejected.push_back(payload); + continue; + } + PrimaryKeyIndexSourceMeta source_meta = std::move(source_meta_result).value(); + auto desired = sources_by_level.find(source_meta.DataLevel()); + if (desired == sources_by_level.end() || desired->second != source_meta.SourceFiles()) { + rejected.push_back(payload); + continue; + } + payloads_by_level[source_meta.DataLevel()].push_back(payload); + payload_metas_by_level[source_meta.DataLevel()].push_back(std::move(source_meta)); + } + + std::vector groups; + std::set covered_levels; + for (const auto& level_payloads : payloads_by_level) { + int32_t level = level_payloads.first; + std::optional group; + if (level_payloads.second.size() == 1) { + group = PkSortedIndexGroup::Create(field_id, index_type, sources_by_level[level], + level_payloads.second[0], + payload_metas_by_level[level][0]); + } + if (group != std::nullopt) { + groups.push_back(std::move(group).value()); + covered_levels.insert(level); + } else { + rejected.insert(rejected.end(), level_payloads.second.begin(), + level_payloads.second.end()); + } + } + + std::vector covered; + std::vector uncovered; + for (const auto& level_sources : sources_by_level) { + auto& target = covered_levels.count(level_sources.first) > 0 ? covered : uncovered; + target.insert(target.end(), level_sources.second.begin(), level_sources.second.end()); + } + return PkSortedBucketIndexState(std::move(groups), std::move(covered), std::move(uncovered), + std::move(rejected)); +} + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h new file mode 100644 index 000000000..2852d61d0 --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h @@ -0,0 +1,75 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/core/index/pksorted/pk_sorted_index_group.h" +#include "paimon/core/io/data_file_meta.h" + +namespace paimon { +/// Immutable source-backed sorted-index state for one field and bucket. +/// +/// Derives the eligible per-level source sets from the active data files, matches the +/// active payloads against them, and keeps the exact validated groups. Payloads whose +/// source metadata cannot be decoded or does not exactly cover its level are rejected; +/// levels without a valid group stay uncovered and must be scanned normally. +class PkSortedBucketIndexState { + public: + static PkSortedBucketIndexState FromActiveDataFiles( + int32_t field_id, const std::string& index_type, + const std::vector>& active_data_files, + const std::vector>& active_payloads); + + const std::vector& Groups() const { + return groups_; + } + + const std::vector& CoveredSourceFiles() const { + return covered_source_files_; + } + + const std::vector& UncoveredSourceFiles() const { + return uncovered_source_files_; + } + + const std::vector>& RejectedPayloads() const { + return rejected_payloads_; + } + + private: + PkSortedBucketIndexState(std::vector groups, + std::vector covered_source_files, + std::vector uncovered_source_files, + std::vector> rejected_payloads) + : groups_(std::move(groups)), + covered_source_files_(std::move(covered_source_files)), + uncovered_source_files_(std::move(uncovered_source_files)), + rejected_payloads_(std::move(rejected_payloads)) {} + + std::vector groups_; + std::vector covered_source_files_; + std::vector uncovered_source_files_; + std::vector> rejected_payloads_; +}; + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp new file mode 100644 index 000000000..d6198d778 --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp @@ -0,0 +1,284 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/index/pksorted/pk_sorted_bucket_index_state.h" + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/core/index/pk/primary_key_index_source_meta.h" +#include "paimon/core/index/pksorted/pk_sorted_index_group.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +class PkSortedBucketIndexStateTest : public ::testing::Test { + public: + std::shared_ptr MakeDataFile(const std::string& file_name, int64_t row_count, + int32_t level, + const std::optional& file_source) const { + return std::make_shared( + file_name, /*file_size=*/1024, row_count, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/1, /*schema_id=*/0, level, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, file_source, /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + } + + /// Builds a payload whose source metadata lists the given sources in the given order. + std::shared_ptr MakePayload( + int32_t field_id, const std::string& index_type, int32_t data_level, + const std::vector& sources, int64_t total_row_count, + int64_t row_range_start, int64_t row_range_end) const { + PrimaryKeyIndexSourceMeta source_meta = + PrimaryKeyIndexSourceMeta::Create(data_level, sources).value(); + std::shared_ptr source_meta_bytes = source_meta.Serialize(pool_.get()).value(); + return MakePayloadWithSourceMetaBytes(field_id, index_type, total_row_count, + row_range_start, row_range_end, source_meta_bytes); + } + + std::shared_ptr MakePayload( + int32_t field_id, const std::string& index_type, int32_t data_level, + const std::vector& sources, int64_t total_row_count) const { + return MakePayload(field_id, index_type, data_level, sources, total_row_count, + /*row_range_start=*/0, /*row_range_end=*/total_row_count - 1); + } + + std::shared_ptr MakePayloadWithSourceMetaBytes( + int32_t field_id, const std::string& index_type, int64_t total_row_count, + int64_t row_range_start, int64_t row_range_end, + const std::shared_ptr& source_meta_bytes) const { + GlobalIndexMeta global_index_meta(row_range_start, row_range_end, field_id, + /*extra_field_ids=*/std::nullopt, + /*index_meta=*/nullptr, source_meta_bytes); + return std::make_shared(index_type, /*file_name=*/"payload.index", + /*file_size=*/2048, total_row_count, + /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, global_index_meta); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(PkSortedBucketIndexStateTest, BuildsGroupWhenPayloadMatchesLevelSources) { + // Files are handed over unsorted; the expected source order is sorted by file name. + std::vector> data_files = { + MakeDataFile("b", 200, 5, FileSource::Compact()), + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::vector expected_sources = {{"a", 100}, {"b", 200}}; + std::shared_ptr payload = + MakePayload(/*field_id=*/7, "btree", /*data_level=*/5, expected_sources, + /*total_row_count=*/300); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_EQ(1, state.Groups().size()); + const PkSortedIndexGroup& group = state.Groups()[0]; + ASSERT_EQ(5, group.DataLevel()); + ASSERT_EQ(300, group.TotalSourceRowCount()); + ASSERT_EQ(expected_sources, group.SourceFiles()); + ASSERT_EQ(payload, group.Payload()); + ASSERT_EQ(expected_sources, state.CoveredSourceFiles()); + ASSERT_TRUE(state.UncoveredSourceFiles().empty()); + ASSERT_TRUE(state.RejectedPayloads().empty()); +} + +TEST_F(PkSortedBucketIndexStateTest, OnlyCompactedFilesAboveLevelZeroAreSources) { + std::vector> data_files = { + MakeDataFile("level0", 10, 0, FileSource::Compact()), + MakeDataFile("appended", 20, 5, FileSource::Append()), + MakeDataFile("unknown_source", 30, 5, std::nullopt), + MakeDataFile("c", 40, 5, FileSource::Compact())}; + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_TRUE(state.CoveredSourceFiles().empty()); + std::vector expected_uncovered = {{"c", 40}}; + ASSERT_EQ(expected_uncovered, state.UncoveredSourceFiles()); + ASSERT_TRUE(state.RejectedPayloads().empty()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithMisorderedSources) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::shared_ptr payload = + MakePayload(7, "btree", 5, {{"b", 200}, {"a", 100}}, 300); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(payload, state.RejectedPayloads()[0]); + ASSERT_TRUE(state.CoveredSourceFiles().empty()); + std::vector expected_uncovered = {{"a", 100}, {"b", 200}}; + ASSERT_EQ(expected_uncovered, state.UncoveredSourceFiles()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithMismatchedSourceRowCount) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::shared_ptr payload = + MakePayload(7, "btree", 5, {{"a", 100}, {"b", 201}}, 301); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(2, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadsCoveringWrongSourceSet) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::shared_ptr missing_source_payload = + MakePayload(7, "btree", 5, {{"a", 100}}, 100); + std::shared_ptr extra_source_payload = + MakePayload(7, "btree", 5, {{"a", 100}, {"b", 200}, {"c", 50}}, 350); + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + 7, "btree", data_files, {missing_source_payload, extra_source_payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(2, state.RejectedPayloads().size()); + ASSERT_TRUE(state.CoveredSourceFiles().empty()); + ASSERT_EQ(2, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsBothPayloadsWhenLevelHasTwoCandidates) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::vector sources = {{"a", 100}, {"b", 200}}; + std::shared_ptr first_payload = MakePayload(7, "btree", 5, sources, 300); + std::shared_ptr second_payload = MakePayload(7, "btree", 5, sources, 300); + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + 7, "btree", data_files, {first_payload, second_payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(2, state.RejectedPayloads().size()); + ASSERT_TRUE(state.CoveredSourceFiles().empty()); + ASSERT_EQ(sources, state.UncoveredSourceFiles()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongFieldId) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::shared_ptr payload = + MakePayload(/*field_id=*/8, "btree", 5, {{"a", 100}}, 100); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(1, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongIndexType) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::shared_ptr payload = MakePayload(7, "bitmap", 5, {{"a", 100}}, 100); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(1, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongRowRange) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::vector sources = {{"a", 100}, {"b", 200}}; + // The exclusive end row 300 violates the required inclusive range [0, 299]. + std::shared_ptr payload = MakePayload(7, "btree", 5, sources, 300, + /*row_range_start=*/0, + /*row_range_end=*/300); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(sources, state.UncoveredSourceFiles()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongRowCount) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + std::vector sources = {{"a", 100}, {"b", 200}}; + // The row range is valid but the payload row count 299 differs from the 300 source rows. + std::shared_ptr payload = MakePayload(7, "btree", 5, sources, + /*total_row_count=*/299, + /*row_range_start=*/0, + /*row_range_end=*/299); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(sources, state.UncoveredSourceFiles()); +} + +TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithCorruptSourceMeta) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact())}; + // Version 1 followed by a truncated data level. + std::shared_ptr corrupt_source_meta = + std::make_shared(std::string("\x00\x00\x00\x01\x00\x00", 6), pool_.get()); + std::shared_ptr payload = + MakePayloadWithSourceMetaBytes(7, "btree", /*total_row_count=*/300, /*row_range_start=*/0, + /*row_range_end=*/299, corrupt_source_meta); + PkSortedBucketIndexState state = + PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); + ASSERT_TRUE(state.Groups().empty()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(payload, state.RejectedPayloads()[0]); + ASSERT_EQ(2, state.UncoveredSourceFiles().size()); +} + +TEST_F(PkSortedBucketIndexStateTest, KeepsValidLevelAndLeavesBrokenLevelUncovered) { + std::vector> data_files = { + MakeDataFile("c", 50, 4, FileSource::Compact()), + MakeDataFile("b", 200, 5, FileSource::Compact()), + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::shared_ptr valid_payload = MakePayload(7, "btree", 4, {{"c", 50}}, 50); + std::shared_ptr broken_payload = + MakePayload(7, "btree", 5, {{"a", 100}, {"b", 999}}, 1099); + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + 7, "btree", data_files, {valid_payload, broken_payload}); + ASSERT_EQ(1, state.Groups().size()); + ASSERT_EQ(4, state.Groups()[0].DataLevel()); + ASSERT_EQ(valid_payload, state.Groups()[0].Payload()); + std::vector expected_covered = {{"c", 50}}; + ASSERT_EQ(expected_covered, state.CoveredSourceFiles()); + std::vector expected_uncovered = {{"a", 100}, {"b", 200}}; + ASSERT_EQ(expected_uncovered, state.UncoveredSourceFiles()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(broken_payload, state.RejectedPayloads()[0]); +} + +} // namespace paimon::test diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp new file mode 100644 index 000000000..c0052d8b1 --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp @@ -0,0 +1,54 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/index/pksorted/pk_sorted_index_group.h" + +#include +#include + +namespace paimon { +std::optional PkSortedIndexGroup::Create( + int32_t field_id, const std::string& index_type, + const std::vector& expected_sources, + const std::shared_ptr& payload, + const PrimaryKeyIndexSourceMeta& payload_source_meta) { + if (payload == nullptr || expected_sources.empty()) { + return std::nullopt; + } + int64_t source_row_count = 0; + std::set source_names; + for (const PrimaryKeyIndexSourceFile& source_file : expected_sources) { + if (!source_names.insert(source_file.file_name).second) { + return std::nullopt; + } + if (__builtin_add_overflow(source_row_count, source_file.row_count, &source_row_count)) { + return std::nullopt; + } + } + + const std::optional& meta = payload->GetGlobalIndexMeta(); + if (payload_source_meta.SourceFiles() != expected_sources || + index_type != payload->IndexType() || meta == std::nullopt || + meta.value().index_field_id != field_id || meta.value().row_range_start != 0 || + meta.value().row_range_end != source_row_count - 1 || + payload->RowCount() != source_row_count) { + return std::nullopt; + } + return PkSortedIndexGroup(payload_source_meta.DataLevel(), expected_sources, payload, + source_row_count); +} + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_group.h b/src/paimon/core/index/pksorted/pk_sorted_index_group.h new file mode 100644 index 000000000..b9cb85cff --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_index_group.h @@ -0,0 +1,77 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/core/index/pk/primary_key_index_source_meta.h" + +namespace paimon { +/// The single validated payload group which indexes one complete data level. +/// +/// A group is only created when the payload provably covers the current active source set +/// of its data level: exactly one payload, unique source names, source order / file names / +/// row counts identical to the expected level sources, matching index type and field id, +/// a row range of exactly `[0, total source rows - 1]` and a payload row count equal to the +/// source row count sum. Anything else must be treated as uncovered. +class PkSortedIndexGroup { + public: + /// Validates one payload against the expected level sources; returns `std::nullopt` + /// when any coverage condition fails. + static std::optional Create( + int32_t field_id, const std::string& index_type, + const std::vector& expected_sources, + const std::shared_ptr& payload, + const PrimaryKeyIndexSourceMeta& payload_source_meta); + + int32_t DataLevel() const { + return data_level_; + } + + const std::vector& SourceFiles() const { + return source_files_; + } + + const std::shared_ptr& Payload() const { + return payload_; + } + + int64_t TotalSourceRowCount() const { + return total_source_row_count_; + } + + private: + PkSortedIndexGroup(int32_t data_level, std::vector source_files, + std::shared_ptr payload, int64_t total_source_row_count) + : data_level_(data_level), + source_files_(std::move(source_files)), + payload_(std::move(payload)), + total_source_row_count_(total_source_row_count) {} + + int32_t data_level_; + std::vector source_files_; + std::shared_ptr payload_; + int64_t total_source_row_count_; +}; + +} // namespace paimon From 2995d6f597d4359f652f4a6c8f761711e82bf7be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=8B=87?= Date: Tue, 11 Aug 2026 02:25:57 -0400 Subject: [PATCH 02/14] feat(core): plan and read primary-key sorted index groups in batch scans Wire the source-backed scalar indexes into ordinary batch scans of primary-key tables, mirroring the Java release-2.0.0 planner: organize same-snapshot data splits and index manifest ADD entries into validated groups, evaluate the indexed part of the scan predicate once per group with a query cache, localize group ordinals to per-file physical row positions by the source row-count prefix, and convert results to indexed splits with the 4096-range fragmentation guard, deletion files kept aligned by file index, and per-file fallback on any untrusted state. The raw read path accepts file-local row ranges (intersected with file index selection, deletion vectors subtracted) and KeyValueTableRead routes indexed splits through it; the reader still applies the complete original predicate. AND may narrow partially, OR is used only when every branch is evaluable, and redundant IS NOT NULL leaves are pruned under AND. Only the BTree payload reader is wired; bitmap / vector / full-text definitions are recognized and conservatively fall back to normal scans. Gated by global-index.enabled (default true). Integration tests arrive with the payload builder in the next part. part of #192 --- src/paimon/CMakeLists.txt | 3 + .../core/operation/raw_file_split_read.cpp | 54 +- .../core/operation/raw_file_split_read.h | 12 +- .../table/source/key_value_table_read.cpp | 21 +- .../source/primary_key_index_batch_scan.cpp | 265 ++++++++ .../source/primary_key_index_batch_scan.h | 70 +++ .../primary_key_sorted_index_result.cpp | 125 ++++ .../source/primary_key_sorted_index_result.h | 47 ++ .../source/primary_key_sorted_index_scan.cpp | 564 ++++++++++++++++++ .../source/primary_key_sorted_index_scan.h | 183 ++++++ .../table/source/snapshot/snapshot_reader.h | 4 + src/paimon/core/table/source/table_scan.cpp | 22 +- 12 files changed, 1360 insertions(+), 10 deletions(-) create mode 100644 src/paimon/core/table/source/primary_key_index_batch_scan.cpp create mode 100644 src/paimon/core/table/source/primary_key_index_batch_scan.h create mode 100644 src/paimon/core/table/source/primary_key_sorted_index_result.cpp create mode 100644 src/paimon/core/table/source/primary_key_sorted_index_result.h create mode 100644 src/paimon/core/table/source/primary_key_sorted_index_scan.cpp create mode 100644 src/paimon/core/table/source/primary_key_sorted_index_scan.h diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 4971fa751..27455e98a 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -403,6 +403,9 @@ set(PAIMON_CORE_SRCS core/table/source/table_read.cpp core/table/source/table_scan.cpp core/table/source/data_evolution_batch_scan.cpp + core/table/source/primary_key_sorted_index_scan.cpp + core/table/source/primary_key_sorted_index_result.cpp + core/table/source/primary_key_index_batch_scan.cpp core/table/system/audit_log_system_table.cpp core/table/system/binlog_system_table.cpp core/table/system/global_system_tables.cpp diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index eabe84268..4d14b537d 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -18,6 +18,7 @@ #include "paimon/core/operation/raw_file_split_read.h" +#include #include #include #include @@ -32,6 +33,7 @@ #include "paimon/core/core_options.h" #include "paimon/core/deletionvectors/bitmap_deletion_vector.h" #include "paimon/core/deletionvectors/deletion_vector.h" +#include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/io/file_index_evaluator.h" #include "paimon/core/operation/internal_read_context.h" @@ -64,6 +66,21 @@ RawFileSplitRead::RawFileSplitRead(const std::shared_ptr& Result> RawFileSplitRead::CreateReader( const std::shared_ptr& split) { + if (auto indexed_split = std::dynamic_pointer_cast(split)) { + PAIMON_RETURN_NOT_OK(indexed_split->Validate()); + const std::shared_ptr& inner_split = indexed_split->GetDataSplit(); + auto inner_split_impl = std::dynamic_pointer_cast(inner_split); + if (!inner_split_impl) { + return Status::Invalid("cannot cast indexed inner split to data_split"); + } + if (inner_split_impl->DataFiles().size() != 1) { + return Status::Invalid( + "indexed splits with file-local row ranges must contain exactly one file"); + } + return CreateReader(inner_split_impl->Partition(), inner_split_impl->Bucket(), + inner_split_impl->DataFiles(), inner_split_impl->DeletionFiles(), + indexed_split->RowRanges()); + } auto data_split = std::dynamic_pointer_cast(split); if (!data_split) { return Status::Invalid("cannot cast split to data_split in RawFileSplitRead"); @@ -75,7 +92,7 @@ Result> RawFileSplitRead::CreateReader( Result> RawFileSplitRead::CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& data_files, - DeletionVector::Factory dv_factory) { + DeletionVector::Factory dv_factory, const std::optional>& local_row_ranges) { const auto& predicate = context_->GetPredicate(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, path_factory_->CreateDataFilePathFactory(partition, bucket)); @@ -83,7 +100,7 @@ Result> RawFileSplitRead::CreateReader( PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, raw_read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory, + local_row_ranges, data_file_path_factory, /*extra_format_options=*/{})); auto raw_readers = @@ -98,10 +115,19 @@ Result> RawFileSplitRead::CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& data_files, const std::vector>& deletion_files) { + return CreateReader(partition, bucket, data_files, deletion_files, + /*local_row_ranges=*/std::nullopt); +} + +Result> RawFileSplitRead::CreateReader( + const BinaryRow& partition, int32_t bucket, + const std::vector>& data_files, + const std::vector>& deletion_files, + const std::optional>& local_row_ranges) { auto dv_factory = DeletionVector::CreateFactory( options_.GetFileSystem(), DeletionVector::CreateDeletionFileMap(data_files, deletion_files), pool_); - return CreateReader(partition, bucket, data_files, dv_factory); + return CreateReader(partition, bucket, data_files, dv_factory, local_row_ranges); } Result RawFileSplitRead::Match(const std::shared_ptr& split, @@ -152,6 +178,28 @@ Result> RawFileSplitRead::ApplyIndexAndDvReader PAIMON_ASSIGN_OR_RAISE(selection, bitmap_file_index->GetBitmap()); } + // narrow the selection to the file-local row positions of an indexed split + std::optional ranges_selection; + if (ranges != std::nullopt) { + RoaringBitmap32 ranges_bitmap; + for (const Range& range : ranges.value()) { + if (range.from < 0 || range.to < range.from || + range.to >= std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("Invalid file-local row range [{}, {}] for file {}.", range.from, + range.to, file->file_name)); + } + ranges_bitmap.AddRange(static_cast(range.from), + static_cast(range.to + 1)); + } + if (selection != nullptr) { + ranges_selection = RoaringBitmap32::And(*selection, ranges_bitmap); + } else { + ranges_selection = std::move(ranges_bitmap); + } + selection = &ranges_selection.value(); + } + // prepare deletion bitmap for deletion vector std::shared_ptr deletion_vector; if (dv_factory) { diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 1580cd848..35fa6576a 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -65,16 +65,26 @@ class RawFileSplitRead : public AbstractSplitRead { const std::shared_ptr& memory_pool, const std::shared_ptr& executor); + /// Also accepts an `IndexedSplit` over a single-file data split, in which case its row + /// ranges narrow the read to the given file-local physical positions. Result> CreateReader(const std::shared_ptr& split) override; Result> CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& files, const std::vector>& deletion_files); + /// Reads with an optional selection of file-local row positions. The ranges apply to + /// every file of the split, so callers pass them only for single-file splits. Result> CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& files, - DeletionVector::Factory dv_factory); + const std::vector>& deletion_files, + const std::optional>& local_row_ranges); + + Result> CreateReader( + const BinaryRow& partition, int32_t bucket, + const std::vector>& files, DeletionVector::Factory dv_factory, + const std::optional>& local_row_ranges = std::nullopt); Result Match(const std::shared_ptr& split, bool force_keep_delete) const override; diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp index 23b902260..ad085fb3f 100644 --- a/src/paimon/core/table/source/key_value_table_read.cpp +++ b/src/paimon/core/table/source/key_value_table_read.cpp @@ -21,6 +21,7 @@ #include +#include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/operation/merge_file_split_read.h" #include "paimon/core/operation/raw_file_split_read.h" #include "paimon/core/table/source/data_split_impl.h" @@ -74,7 +75,25 @@ void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) { Result> KeyValueTableRead::CreateReader( const std::shared_ptr& split) { - auto data_split = std::dynamic_pointer_cast(split); + std::shared_ptr dispatch_split = split; + if (auto indexed_split = std::dynamic_pointer_cast(split)) { + // A primary-key sorted-index split narrows one raw-readable file to file-local row + // positions. If the raw read cannot serve the inner split, fall back to reading the + // whole file: the index only narrows the scan, so the unnarrowed read stays correct. + const std::shared_ptr& inner_split = indexed_split->GetDataSplit(); + for (const auto& read : split_reads_) { + auto* raw_read = dynamic_cast(read.get()); + if (raw_read == nullptr) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(bool matched, read->Match(inner_split, force_keep_delete_)); + if (matched) { + return read->CreateReader(indexed_split); + } + } + dispatch_split = inner_split; + } + auto data_split = std::dynamic_pointer_cast(dispatch_split); if (!data_split) { return Status::Invalid("split cannot be casted to DataSplit"); } diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp new file mode 100644 index 000000000..b5dbb153a --- /dev/null +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp @@ -0,0 +1,265 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/source/primary_key_index_batch_scan.h" + +#include +#include + +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/table/source/plan_impl.h" +#include "paimon/core/table/source/primary_key_sorted_index_result.h" +#include "paimon/core/table/source/primary_key_sorted_index_scan.h" +#include "paimon/core/table/source/snapshot/snapshot_reader.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/predicate_builder.h" + +namespace paimon { +namespace { +/// Restricts a predicate to leaves over the indexed fields: an AND keeps its convertible +/// children, an OR is only kept when every child is convertible, and everything else is +/// dropped. A null return means no part of the predicate can use the index. +Result> ProjectToIndexedFields( + const std::shared_ptr& predicate, const std::set& indexed_fields) { + if (predicate == nullptr) { + return std::shared_ptr(nullptr); + } + if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { + if (indexed_fields.count(leaf_predicate->FieldName()) > 0) { + return predicate; + } + return std::shared_ptr(nullptr); + } + auto compound_predicate = std::dynamic_pointer_cast(predicate); + if (compound_predicate == nullptr) { + return std::shared_ptr(nullptr); + } + bool is_and = compound_predicate->GetFunction().GetType() == Function::Type::AND; + bool is_or = compound_predicate->GetFunction().GetType() == Function::Type::OR; + if (!is_and && !is_or) { + return std::shared_ptr(nullptr); + } + std::vector> converted_children; + for (const std::shared_ptr& child : compound_predicate->Children()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converted_child, + ProjectToIndexedFields(child, indexed_fields)); + if (converted_child != nullptr) { + converted_children.push_back(std::move(converted_child)); + } else if (is_or) { + return std::shared_ptr(nullptr); + } + } + if (converted_children.empty()) { + return std::shared_ptr(nullptr); + } + if (converted_children.size() == 1) { + return converted_children[0]; + } + if (is_and) { + return PredicateBuilder::And(converted_children); + } + return PredicateBuilder::Or(converted_children); +} + +void FlattenChildren(const std::shared_ptr& compound_predicate, + std::vector>* flattened) { + for (const std::shared_ptr& child : compound_predicate->Children()) { + auto compound_child = std::dynamic_pointer_cast(child); + if (compound_child != nullptr && compound_child->GetFunction().GetType() == + compound_predicate->GetFunction().GetType()) { + FlattenChildren(compound_child, flattened); + } else { + flattened->push_back(child); + } + } +} + +/// A predicate is null-rejecting when it cannot match a row whose tested field is null. +/// Under SQL three-valued logic every comparison and match predicate rejects null; only +/// IS NULL accepts it, and IS NOT NULL is the predicate being pruned. +bool IsNullRejecting(const std::shared_ptr& predicate) { + auto leaf_predicate = std::dynamic_pointer_cast(predicate); + if (leaf_predicate == nullptr) { + return false; + } + switch (leaf_predicate->GetFunction().GetType()) { + case Function::Type::EQUAL: + case Function::Type::NOT_EQUAL: + case Function::Type::GREATER_THAN: + case Function::Type::GREATER_OR_EQUAL: + case Function::Type::LESS_THAN: + case Function::Type::LESS_OR_EQUAL: + case Function::Type::IN: + case Function::Type::NOT_IN: + case Function::Type::STARTS_WITH: + case Function::Type::ENDS_WITH: + case Function::Type::CONTAINS: + case Function::Type::LIKE: + return true; + default: + return false; + } +} + +bool IsIsNotNull(const std::shared_ptr& predicate) { + auto leaf_predicate = std::dynamic_pointer_cast(predicate); + return leaf_predicate != nullptr && + leaf_predicate->GetFunction().GetType() == Function::Type::IS_NOT_NULL; +} + +/// Flattens nested same-function compounds and, inside an AND, removes `f IS NOT NULL` +/// leaves made redundant by a null-rejecting sibling on the same field. Pruning must not +/// consider `f IS NULL` as constraining: dropping IS NOT NULL from +/// "f IS NULL AND f IS NOT NULL" would turn the empty result into the set of null rows. +Result> NormalizePredicate(const std::shared_ptr& predicate) { + auto compound_predicate = std::dynamic_pointer_cast(predicate); + if (compound_predicate == nullptr) { + return predicate; + } + std::vector> children; + FlattenChildren(compound_predicate, &children); + + bool is_and = compound_predicate->GetFunction().GetType() == Function::Type::AND; + if (is_and) { + std::set constrained_fields; + for (const std::shared_ptr& child : children) { + if (IsNullRejecting(child)) { + constrained_fields.insert( + std::dynamic_pointer_cast(child)->FieldName()); + } + } + if (!constrained_fields.empty()) { + std::vector> pruned; + pruned.reserve(children.size()); + for (const std::shared_ptr& child : children) { + if (IsIsNotNull(child) && + constrained_fields.count( + std::dynamic_pointer_cast(child)->FieldName()) > 0) { + continue; + } + pruned.push_back(child); + } + children = std::move(pruned); + } + } + + std::vector> normalized_children; + normalized_children.reserve(children.size()); + for (const std::shared_ptr& child : children) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_child, + NormalizePredicate(child)); + normalized_children.push_back(std::move(normalized_child)); + } + if (normalized_children.size() == 1) { + return normalized_children[0]; + } + if (is_and) { + return PredicateBuilder::And(normalized_children); + } + return PredicateBuilder::Or(normalized_children); +} +} // namespace + +Result> PrimaryKeyIndexBatchScan::Create( + const std::shared_ptr& snapshot_reader, + std::unique_ptr&& batch_scan, + const std::shared_ptr& table_schema, + const std::shared_ptr& path_factory, const CoreOptions& core_options, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + return std::unique_ptr(new PrimaryKeyIndexBatchScan( + snapshot_reader, std::move(batch_scan), table_schema, path_factory, core_options, pool, + definitions.ScalarDefinitions())); +} + +Result> PrimaryKeyIndexBatchScan::CreatePlan() { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_plan, batch_scan_->CreatePlan()); + if (!core_options_.GlobalIndexEnabled() || scalar_definitions_.empty() || + data_plan->SnapshotId() == std::nullopt || data_plan->Splits().empty()) { + return data_plan; + } + + std::set indexed_fields; + std::set indexed_field_ids; + for (const PrimaryKeyIndexDefinition& definition : scalar_definitions_) { + indexed_fields.insert(definition.Column()); + indexed_field_ids.insert(definition.FieldId()); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_predicate, + ProjectToIndexedFields(batch_scan_->GetNonPartitionPredicate(), indexed_fields)); + if (index_predicate == nullptr) { + return data_plan; + } + PAIMON_ASSIGN_OR_RAISE(index_predicate, NormalizePredicate(index_predicate)); + + std::vector> data_splits; + data_splits.reserve(data_plan->Splits().size()); + for (const std::shared_ptr& split : data_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (data_split == nullptr || data_split->IsStreaming()) { + return data_plan; + } + data_splits.push_back(std::move(data_split)); + } + + int64_t snapshot_id = data_plan->SnapshotId().value(); + const std::shared_ptr& snapshot_manager = + snapshot_reader_->GetSnapshotManager(); + Result snapshot_result = snapshot_manager->LoadSnapshot(snapshot_id); + if (!snapshot_result.ok()) { + return data_plan; + } + + const std::unique_ptr& index_file_handler = + snapshot_reader_->GetIndexFileHandler(); + if (index_file_handler == nullptr) { + return data_plan; + } + std::function(const IndexManifestEntry&)> entry_filter = + [&indexed_field_ids](const IndexManifestEntry& entry) -> Result { + if (!(entry.kind == FileKind::Add()) || entry.index_file == nullptr) { + return false; + } + const std::optional& meta = entry.index_file->GetGlobalIndexMeta(); + return meta != std::nullopt && meta.value().source_meta != nullptr && + indexed_field_ids.count(meta.value().index_field_id) > 0; + }; + PAIMON_ASSIGN_OR_RAISE(std::vector index_entries, + index_file_handler->Scan(snapshot_result.value(), entry_filter)); + + PAIMON_ASSIGN_OR_RAISE(PrimaryKeySortedIndexScan::Plan index_plan, + PrimaryKeySortedIndexScan::CreatePlan( + snapshot_id, data_splits, scalar_definitions_, index_entries)); + PrimaryKeySortedIndexScan::ReaderFactory reader_factory = + PrimaryKeySortedIndexScan::MakeReaderFactory( + core_options_.GetFileSystem(), std::make_shared(path_factory_), + table_schema_, core_options_.ToMap(), pool_); + PAIMON_ASSIGN_OR_RAISE( + PrimaryKeySortedIndexScan::EvaluatedPlan evaluated_plan, + PrimaryKeySortedIndexScan::Evaluate(index_plan, table_schema_, index_predicate, + scalar_definitions_, reader_factory)); + PAIMON_ASSIGN_OR_RAISE(std::vector> splits, + PrimaryKeySortedIndexResult::ToSplits(evaluated_plan)); + return std::make_shared(data_plan->SnapshotId(), splits); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.h b/src/paimon/core/table/source/primary_key_index_batch_scan.h new file mode 100644 index 000000000..d330de703 --- /dev/null +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.h @@ -0,0 +1,70 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/core/index/pk/primary_key_index_definition.h" +#include "paimon/core/table/source/abstract_table_scan.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/result.h" + +namespace paimon { +/// Batch scan for primary-key tables with source-backed scalar index definitions. +/// +/// Wraps the ordinary batch scan: the data plan is computed first, then the part of the +/// scan predicate that touches indexed fields is evaluated against the validated payload +/// groups of the plan's snapshot, and covered files are narrowed to indexed splits with +/// file-local row ranges. Files without trustworthy coverage keep their normal scan; the +/// reader still applies deletion vectors and the complete original predicate. +class PrimaryKeyIndexBatchScan : public AbstractTableScan { + public: + static Result> Create( + const std::shared_ptr& snapshot_reader, + std::unique_ptr&& batch_scan, + const std::shared_ptr& table_schema, + const std::shared_ptr& path_factory, const CoreOptions& core_options, + const std::shared_ptr& pool); + + Result> CreatePlan() override; + + private: + PrimaryKeyIndexBatchScan(const std::shared_ptr& snapshot_reader, + std::unique_ptr&& batch_scan, + const std::shared_ptr& table_schema, + const std::shared_ptr& path_factory, + const CoreOptions& core_options, + const std::shared_ptr& pool, + std::vector scalar_definitions) + : AbstractTableScan(core_options, snapshot_reader), + batch_scan_(std::move(batch_scan)), + table_schema_(table_schema), + path_factory_(path_factory), + pool_(pool), + scalar_definitions_(std::move(scalar_definitions)) {} + + std::unique_ptr batch_scan_; + std::shared_ptr table_schema_; + std::shared_ptr path_factory_; + std::shared_ptr pool_; + std::vector scalar_definitions_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp new file mode 100644 index 000000000..c0ebfadb4 --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp @@ -0,0 +1,125 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/source/primary_key_sorted_index_result.h" + +#include +#include +#include +#include + +#include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/table/source/deletion_file.h" + +namespace paimon { +namespace { +Result> ToSingleFileSplit( + const PrimaryKeySortedIndexScan::FilePlan& file) { + const std::shared_ptr& source = file.SourceSplit(); + std::vector> data_files{file.DataFile()}; + DataSplitImpl::Builder builder(source->Partition(), source->Bucket(), source->BucketPath(), + std::move(data_files)); + builder.WithSnapshot(source->SnapshotId()) + .WithTotalBuckets(source->TotalBuckets()) + .IsStreaming(false) + .RawConvertible(source->RawConvertible()); + if (!source->DeletionFiles().empty()) { + builder.WithDataDeletionFiles({source->DeletionFiles()[file.FileIndex()]}); + } + return builder.Build(); +} + +/// Converts sorted file-local positions to merged ranges. Returns `std::nullopt` when a +/// position is invalid or the result is over-fragmented, in which case the file must fall +/// back to a normal scan. +Result>> ToRanges(const GlobalIndexResult& result, + int64_t row_count) { + std::vector ranges; + int64_t from = -1; + int64_t to = -1; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + result.CreateIterator()); + while (iterator->HasNext()) { + int64_t position = iterator->Next(); + if (position < 0 || position >= row_count || + position >= std::numeric_limits::max()) { + return std::optional>(); + } + if (from < 0) { + from = position; + } else if (position != to + 1) { + if (ranges.size() >= + static_cast(PrimaryKeySortedIndexResult::kMaxIndexedRangesPerFile)) { + return std::optional>(); + } + ranges.emplace_back(from, to); + from = position; + } + to = position; + } + if (ranges.size() >= + static_cast(PrimaryKeySortedIndexResult::kMaxIndexedRangesPerFile)) { + return std::optional>(); + } + ranges.emplace_back(from, to); + return std::optional>(std::move(ranges)); +} +} // namespace + +Result>> PrimaryKeySortedIndexResult::ToSplits( + const PrimaryKeySortedIndexScan::EvaluatedPlan& evaluated_plan) { + std::vector> splits; + std::set preserved_non_raw_splits; + for (const PrimaryKeySortedIndexScan::EvaluatedFile& evaluated_file : evaluated_plan.Files()) { + const PrimaryKeySortedIndexScan::FilePlan& file = evaluated_file.File(); + const std::shared_ptr& source_split = file.SourceSplit(); + if (!source_split->RawConvertible()) { + // Splits that cannot be read file by file keep their original shape. + if (preserved_non_raw_splits.insert(source_split.get()).second) { + splits.push_back(source_split); + } + continue; + } + + const std::shared_ptr& result = evaluated_file.IndexResult(); + if (result == nullptr) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fallback_split, + ToSingleFileSplit(file)); + splits.push_back(std::move(fallback_split)); + continue; + } + + PAIMON_ASSIGN_OR_RAISE(bool is_empty, result->IsEmpty()); + if (is_empty) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::optional> ranges, + ToRanges(*result, file.DataFile()->row_count)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr single_file_split, + ToSingleFileSplit(file)); + if (ranges == std::nullopt) { + // The index returned an invalid or over-fragmented row position set; fall back + // to a normal scan for this file. + splits.push_back(std::move(single_file_split)); + } else { + splits.push_back(std::make_shared( + std::move(single_file_split), std::move(ranges).value(), std::vector())); + } + } + return splits; +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_result.h b/src/paimon/core/table/source/primary_key_sorted_index_result.h new file mode 100644 index 000000000..bd6ed262d --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_result.h @@ -0,0 +1,47 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/core/table/source/primary_key_sorted_index_scan.h" +#include "paimon/table/source/split.h" + +namespace paimon { +/// Converts an evaluated primary-key sorted-index plan into scan splits addressed by +/// physical data-file row positions. +/// +/// Files whose index result is missing or untrustworthy keep a normal single-file scan, +/// files with an empty result are omitted, and files with a valid result become indexed +/// splits carrying their file-local row ranges next to the aligned deletion file. Source +/// splits that are not raw-convertible are preserved unchanged. +class PrimaryKeySortedIndexResult { + public: + /// The fragmentation guard of the Java implementation: a file whose index result needs + /// more ranges falls back to a normal scan. + static constexpr int32_t kMaxIndexedRangesPerFile = 4096; + + PrimaryKeySortedIndexResult() = delete; + ~PrimaryKeySortedIndexResult() = delete; + + static Result>> ToSplits( + const PrimaryKeySortedIndexScan::EvaluatedPlan& evaluated_plan); +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp new file mode 100644 index 000000000..fe7689c3e --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -0,0 +1,564 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/source/primary_key_sorted_index_scan.h" + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "fmt/format.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/global_index/global_index_evaluator_impl.h" +#include "paimon/core/index/pksorted/pk_sorted_bucket_index_state.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/predicate/predicate_utils.h" + +namespace paimon { +namespace { +using BucketKey = std::pair; + +enum class QueryOperation { + IS_NOT_NULL, + IS_NULL, + EQUAL, + NOT_EQUAL, + LESS_THAN, + LESS_OR_EQUAL, + GREATER_THAN, + GREATER_OR_EQUAL, + IN, + NOT_IN, + STARTS_WITH, + ENDS_WITH, + CONTAINS, + LIKE, +}; + +struct QueryKey { + QueryOperation operation; + std::vector literals; + + bool operator==(const QueryKey& other) const { + return operation == other.operation && literals == other.literals; + } +}; + +/// Shares one group payload reader and its group-scope query results across all source +/// files of the group; localizes group ordinals to file-local physical positions using the +/// ordered source row-count prefix. +class SharedGroupReader { + public: + using UnderlyingReaderFactory = std::function>()>; + + SharedGroupReader(const std::shared_ptr& group, + UnderlyingReaderFactory reader_factory) + : group_(group), reader_factory_(std::move(reader_factory)) { + const std::vector& source_files = group->SourceFiles(); + source_offsets_.reserve(source_files.size() + 1); + source_offsets_.push_back(0); + for (const PrimaryKeyIndexSourceFile& source_file : source_files) { + source_offsets_.push_back(source_offsets_.back() + source_file.row_count); + } + } + + const std::shared_ptr& Group() const { + return group_; + } + + /// Runs one group-scope query with caching; equal queries evaluate exactly once. + Result> Query( + const QueryKey& key, + const std::function>(GlobalIndexReader*)>& + query) { + for (const auto& cached : query_cache_) { + if (cached.first == key) { + if (!cached.second.status.ok()) { + return cached.second.status; + } + return cached.second.result; + } + } + Result> result = RunQuery(query); + CachedQuery cached_query; + if (result.ok()) { + cached_query.result = result.value(); + } else { + cached_query.status = result.status(); + } + query_cache_.emplace_back(key, cached_query); + return result; + } + + /// Restricts one group-scope result to the local row positions of `source_index`. + /// Any out-of-range group ordinal fails the localization so that every covered file + /// of this group falls back to a normal scan; a poison marker would not survive the + /// AND/OR combination of results from other indexes. + Result> Localize( + const std::shared_ptr& result, size_t source_index) { + if (result == nullptr) { + return std::shared_ptr(nullptr); + } + assert(source_index + 1 < source_offsets_.size()); + auto localized = localized_cache_.find(result.get()); + if (localized == localized_cache_.end()) { + PAIMON_ASSIGN_OR_RAISE(std::vector> partitions, + PartitionBySource(result)); + localized = localized_cache_.emplace(result.get(), std::move(partitions)).first; + } + return localized->second[source_index]; + } + + private: + struct CachedQuery { + Status status; + std::shared_ptr result; + }; + + Result> RunQuery( + const std::function>(GlobalIndexReader*)>& + query) { + if (!reader_status_.ok()) { + return reader_status_; + } + if (reader_ == nullptr) { + Result> reader_result = reader_factory_(); + if (!reader_result.ok()) { + reader_status_ = reader_result.status(); + return reader_status_; + } + reader_ = reader_result.value(); + if (reader_ == nullptr) { + // The index type has no usable reader; keep normal scan semantics. + return std::shared_ptr(nullptr); + } + } + return query(reader_.get()); + } + + Result>> PartitionBySource( + const std::shared_ptr& result) { + size_t source_count = source_offsets_.size() - 1; + std::vector partitions(source_count); + int64_t total_row_count = source_offsets_.back(); + size_t source_index = 0; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + result->CreateIterator()); + while (iterator->HasNext()) { + int64_t position = iterator->Next(); + if (position < 0 || position >= total_row_count) { + return Status::Invalid(fmt::format( + "Sorted index returned group ordinal {} outside the source row range " + "[0, {}).", + position, total_row_count)); + } + while (position >= source_offsets_[source_index + 1]) { + source_index++; + } + partitions[source_index].Add(position - source_offsets_[source_index]); + } + std::vector> localized; + localized.reserve(source_count); + for (RoaringBitmap64& partition : partitions) { + auto bitmap = std::make_shared(std::move(partition)); + localized.push_back(std::make_shared( + [bitmap]() -> Result { return *bitmap; })); + } + return localized; + } + + std::shared_ptr group_; + UnderlyingReaderFactory reader_factory_; + std::vector source_offsets_; + std::vector> query_cache_; + std::unordered_map>> + localized_cache_; + std::shared_ptr reader_; + Status reader_status_; +}; + +/// Restricts merged source-group ordinals to one source file's local row positions. +class FileLocalGroupReader : public GlobalIndexReader { + public: + FileLocalGroupReader(std::shared_ptr shared_reader, size_t source_index) + : shared_reader_(std::move(shared_reader)), source_index_(source_index) {} + + Result> VisitIsNotNull() override { + return Query({QueryOperation::IS_NOT_NULL, {}}, + [](GlobalIndexReader* reader) { return reader->VisitIsNotNull(); }); + } + + Result> VisitIsNull() override { + return Query({QueryOperation::IS_NULL, {}}, + [](GlobalIndexReader* reader) { return reader->VisitIsNull(); }); + } + + Result> VisitEqual(const Literal& literal) override { + return Query({QueryOperation::EQUAL, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitEqual(literal); }); + } + + Result> VisitNotEqual(const Literal& literal) override { + return Query({QueryOperation::NOT_EQUAL, {literal}}, [&literal](GlobalIndexReader* reader) { + return reader->VisitNotEqual(literal); + }); + } + + Result> VisitLessThan(const Literal& literal) override { + return Query({QueryOperation::LESS_THAN, {literal}}, [&literal](GlobalIndexReader* reader) { + return reader->VisitLessThan(literal); + }); + } + + Result> VisitLessOrEqual(const Literal& literal) override { + return Query( + {QueryOperation::LESS_OR_EQUAL, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitLessOrEqual(literal); }); + } + + Result> VisitGreaterThan(const Literal& literal) override { + return Query( + {QueryOperation::GREATER_THAN, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitGreaterThan(literal); }); + } + + Result> VisitGreaterOrEqual( + const Literal& literal) override { + return Query( + {QueryOperation::GREATER_OR_EQUAL, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitGreaterOrEqual(literal); }); + } + + Result> VisitIn( + const std::vector& literals) override { + return Query({QueryOperation::IN, literals}, + [&literals](GlobalIndexReader* reader) { return reader->VisitIn(literals); }); + } + + Result> VisitNotIn( + const std::vector& literals) override { + return Query({QueryOperation::NOT_IN, literals}, [&literals](GlobalIndexReader* reader) { + return reader->VisitNotIn(literals); + }); + } + + Result> VisitStartsWith(const Literal& prefix) override { + return Query({QueryOperation::STARTS_WITH, {prefix}}, [&prefix](GlobalIndexReader* reader) { + return reader->VisitStartsWith(prefix); + }); + } + + Result> VisitEndsWith(const Literal& suffix) override { + return Query({QueryOperation::ENDS_WITH, {suffix}}, [&suffix](GlobalIndexReader* reader) { + return reader->VisitEndsWith(suffix); + }); + } + + Result> VisitContains(const Literal& literal) override { + return Query({QueryOperation::CONTAINS, {literal}}, [&literal](GlobalIndexReader* reader) { + return reader->VisitContains(literal); + }); + } + + Result> VisitLike(const Literal& literal) override { + return Query({QueryOperation::LIKE, {literal}}, + [&literal](GlobalIndexReader* reader) { return reader->VisitLike(literal); }); + } + + Result> VisitVectorSearch( + const std::shared_ptr& vector_search) override { + return Status::Invalid("Primary-key sorted index does not support vector search."); + } + + Result> VisitFullTextSearch( + const std::shared_ptr& full_text_search) override { + return Status::Invalid("Primary-key sorted index does not support full text search."); + } + + bool IsThreadSafe() const override { + return false; + } + + std::string GetIndexType() const override { + return shared_reader_->Group()->Payload()->IndexType(); + } + + private: + Result> Query( + QueryKey key, + const std::function>(GlobalIndexReader*)>& + query) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr group_result, + shared_reader_->Query(key, query)); + return shared_reader_->Localize(group_result, source_index_); + } + + std::shared_ptr shared_reader_; + size_t source_index_; +}; + +Result FindSourceIndex(const PkSortedIndexGroup& group, const DataFileMeta& data_file) { + const std::vector& source_files = group.SourceFiles(); + for (size_t i = 0; i < source_files.size(); i++) { + if (source_files[i].file_name == data_file.file_name && + source_files[i].row_count == data_file.row_count) { + return i; + } + } + return Status::Invalid(fmt::format( + "Data file {} is not covered by its sorted-index source group.", data_file.file_name)); +} +} // namespace + +Result PrimaryKeySortedIndexScan::CreatePlan( + int64_t snapshot_id, const std::vector>& data_splits, + const std::vector& definitions, + const std::vector& index_entries) { + std::unordered_map>> payloads_by_bucket; + for (const IndexManifestEntry& entry : index_entries) { + const std::shared_ptr& payload = entry.index_file; + if (payload == nullptr || !(entry.kind == FileKind::Add())) { + continue; + } + const std::optional& meta = payload->GetGlobalIndexMeta(); + if (meta == std::nullopt || meta.value().source_meta == nullptr) { + continue; + } + payloads_by_bucket[BucketKey(entry.partition, entry.bucket)].push_back(payload); + } + + std::vector scalar_definitions; + for (const PrimaryKeyIndexDefinition& definition : definitions) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE || + definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BITMAP) { + scalar_definitions.push_back(definition); + } + } + + std::unordered_map>> data_files_by_bucket; + for (const std::shared_ptr& split : data_splits) { + if (split == nullptr) { + return Status::Invalid("Primary-key sorted-index scan received a null data split."); + } + if (split->SnapshotId() != snapshot_id) { + return Status::Invalid( + fmt::format("Data split snapshot {} does not match sorted-index scan snapshot {}.", + split->SnapshotId(), snapshot_id)); + } + if (split->IsStreaming()) { + return Status::Invalid("Primary-key sorted-index scan requires batch splits."); + } + if (!split->DeletionFiles().empty() && + split->DeletionFiles().size() != split->DataFiles().size()) { + return Status::Invalid( + "Deletion files must align with data files in a sorted-index split."); + } + std::vector>& data_files = + data_files_by_bucket[BucketKey(split->Partition(), split->Bucket())]; + data_files.insert(data_files.end(), split->DataFiles().begin(), split->DataFiles().end()); + } + + // file name -> field id -> validated group, per bucket. + std::unordered_map< + BucketKey, std::map>>> + groups_by_bucket; + for (const auto& bucket_entry : data_files_by_bucket) { + const BucketKey& bucket = bucket_entry.first; + std::vector> bucket_payloads; + auto payloads_iter = payloads_by_bucket.find(bucket); + if (payloads_iter != payloads_by_bucket.end()) { + bucket_payloads = payloads_iter->second; + } + std::set> active_source_files; + for (const std::shared_ptr& data_file : bucket_entry.second) { + active_source_files.emplace(data_file->file_name, data_file->row_count); + } + std::map>> + groups_by_source; + for (const PrimaryKeyIndexDefinition& definition : scalar_definitions) { + std::vector> definition_payloads; + for (const std::shared_ptr& payload : bucket_payloads) { + const std::optional& meta = payload->GetGlobalIndexMeta(); + if (meta != std::nullopt && definition.IndexType() == payload->IndexType() && + definition.FieldId() == meta.value().index_field_id) { + definition_payloads.push_back(payload); + } + } + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + definition.FieldId(), definition.IndexType(), bucket_entry.second, + definition_payloads); + for (const PkSortedIndexGroup& group : state.Groups()) { + auto shared_group = std::make_shared(group); + for (const PrimaryKeyIndexSourceFile& source_file : group.SourceFiles()) { + if (active_source_files.count({source_file.file_name, source_file.row_count}) == + 0) { + continue; + } + groups_by_source[source_file.file_name][definition.FieldId()] = shared_group; + } + } + } + groups_by_bucket[bucket] = std::move(groups_by_source); + } + + std::vector files; + for (const std::shared_ptr& split : data_splits) { + auto bucket_groups = groups_by_bucket.find(BucketKey(split->Partition(), split->Bucket())); + for (size_t file_index = 0; file_index < split->DataFiles().size(); file_index++) { + const std::shared_ptr& data_file = split->DataFiles()[file_index]; + std::map> groups; + if (bucket_groups != groups_by_bucket.end() && data_file != nullptr) { + auto source_groups = bucket_groups->second.find(data_file->file_name); + if (source_groups != bucket_groups->second.end()) { + groups = source_groups->second; + } + } + files.emplace_back(split, static_cast(file_index), std::move(groups)); + } + } + return Plan(snapshot_id, std::move(files)); +} + +Result PrimaryKeySortedIndexScan::Evaluate( + const Plan& plan, const std::shared_ptr& table_schema, + const std::shared_ptr& predicate, + const std::vector& definitions, + const ReaderFactory& reader_factory) { + std::map definitions_by_field; + for (const PrimaryKeyIndexDefinition& definition : definitions) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE || + definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BITMAP) { + definitions_by_field.emplace(definition.FieldId(), definition); + } + } + + std::unordered_map> + shared_readers; + std::vector files; + files.reserve(plan.Files().size()); + for (const FilePlan& file : plan.Files()) { + GlobalIndexEvaluatorImpl::IndexReadersCreator create_readers = + [&file, &definitions_by_field, &shared_readers, &reader_factory]( + int32_t field_id) -> Result>> { + auto definition_iter = definitions_by_field.find(field_id); + std::shared_ptr group = file.Group(field_id); + if (definition_iter == definitions_by_field.end() || group == nullptr) { + return std::vector>(); + } + auto shared_iter = shared_readers.find(group.get()); + if (shared_iter == shared_readers.end()) { + const PrimaryKeyIndexDefinition& definition = definition_iter->second; + // The shared reader outlives this file plan, so the factory owns a copy of + // the file plan instead of referencing the loop variable. + SharedGroupReader::UnderlyingReaderFactory underlying_factory = + [file_copy = file, definition, group, + &reader_factory]() -> Result> { + return reader_factory(file_copy, definition, *group); + }; + shared_iter = shared_readers + .emplace(group.get(), std::make_shared( + group, std::move(underlying_factory))) + .first; + } + PAIMON_ASSIGN_OR_RAISE(size_t source_index, FindSourceIndex(*group, *file.DataFile())); + std::vector> readers; + readers.push_back( + std::make_shared(shared_iter->second, source_index)); + return readers; + }; + GlobalIndexEvaluatorImpl evaluator(table_schema, create_readers); + Result> result = evaluator.Evaluate(predicate); + if (result.ok()) { + files.emplace_back(file, result.value()); + } else { + // Evaluation failures degrade to a normal scan for this file only. + files.emplace_back(file, nullptr); + } + } + return EvaluatedPlan(plan.SnapshotId(), std::move(files)); +} + +namespace { +class FsGlobalIndexFileReader : public GlobalIndexFileReader { + public: + explicit FsGlobalIndexFileReader(std::shared_ptr file_system) + : file_system_(std::move(file_system)) {} + + Result> GetInputStream( + const std::string& file_path) const override { + return file_system_->Open(file_path); + } + + private: + std::shared_ptr file_system_; +}; +} // namespace + +PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFactory( + const std::shared_ptr& file_system, + const std::shared_ptr& path_factories, + const std::shared_ptr& table_schema, + const std::map& options, const std::shared_ptr& pool) { + auto file_reader = std::make_shared(file_system); + return [path_factories, table_schema, options, pool, file_reader]( + const FilePlan& file, const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + if (definition.GetFamily() != PrimaryKeyIndexDefinition::Family::BTREE) { + // Only the BTree payload reader is wired up; other families keep normal scan + // semantics until their dedicated readers are supported. + return std::shared_ptr(nullptr); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(definition.IndexType(), options)); + if (indexer == nullptr) { + return std::shared_ptr(nullptr); + } + const std::shared_ptr& split = file.SourceSplit(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + path_factories->Get(split->Partition(), split->Bucket())); + const std::shared_ptr& payload = group.Payload(); + const std::optional& payload_meta = payload->GetGlobalIndexMeta(); + if (payload_meta == std::nullopt) { + // Group validation guarantees the metadata; degrade to a normal scan of the + // covered files if it is ever violated, like the Java reader factory. + return Status::Invalid(fmt::format( + "Sorted index payload {} has no global index metadata.", payload->FileName())); + } + std::vector io_metas; + io_metas.emplace_back(path_factory->ToPath(payload), payload->FileSize(), + payload_meta.value().index_meta); + PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(definition.FieldId())); + auto arrow_field = DataField::ConvertDataFieldToArrowField(field); + auto arrow_schema = arrow::schema({arrow_field}); + ArrowSchema c_arrow_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); + ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); + }; +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.h b/src/paimon/core/table/source/primary_key_sorted_index_scan.h new file mode 100644 index 000000000..c59bf8ed6 --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.h @@ -0,0 +1,183 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/core/index/pk/primary_key_index_definition.h" +#include "paimon/core/index/pksorted/pk_sorted_index_group.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/fs/file_system.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/global_index_result.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/predicate.h" +#include "paimon/result.h" + +namespace paimon { +/// Plans and evaluates source-backed primary-key scalar index groups in file-local +/// row-position space. +/// +/// The scan works on one captured snapshot: data splits and index manifest entries must +/// come from the same snapshot. Every active data file is associated with the validated +/// payload group of its (bucket, field, data level); files without a valid group keep an +/// empty group map and later fall back to a normal scan. Evaluation runs the predicate +/// against the group payloads once per group and query, then localizes group ordinals to +/// per-file physical row positions using the ordered source row-count prefix. +class PrimaryKeySortedIndexScan { + public: + PrimaryKeySortedIndexScan() = delete; + ~PrimaryKeySortedIndexScan() = delete; + + /// One active data file and its complete field-local payload groups. + class FilePlan { + public: + FilePlan(std::shared_ptr source_split, int32_t file_index, + std::map> groups) + : source_split_(std::move(source_split)), + file_index_(file_index), + groups_(std::move(groups)) {} + + const std::shared_ptr& SourceSplit() const { + return source_split_; + } + + int32_t FileIndex() const { + return file_index_; + } + + const std::shared_ptr& DataFile() const { + return source_split_->DataFiles()[file_index_]; + } + + std::shared_ptr Group(int32_t field_id) const { + auto iter = groups_.find(field_id); + return iter == groups_.end() ? nullptr : iter->second; + } + + const std::map>& Groups() const { + return groups_; + } + + private: + std::shared_ptr source_split_; + int32_t file_index_; + std::map> groups_; + }; + + /// Immutable groups for all source files in one captured snapshot. + class Plan { + public: + Plan(int64_t snapshot_id, std::vector files) + : snapshot_id_(snapshot_id), files_(std::move(files)) {} + + int64_t SnapshotId() const { + return snapshot_id_; + } + + const std::vector& Files() const { + return files_; + } + + private: + int64_t snapshot_id_; + std::vector files_; + }; + + /// Optional file-local index result; a null result means that the file requires a + /// normal scan. + class EvaluatedFile { + public: + EvaluatedFile(FilePlan file, std::shared_ptr result) + : file_(std::move(file)), result_(std::move(result)) {} + + const FilePlan& File() const { + return file_; + } + + const std::shared_ptr& IndexResult() const { + return result_; + } + + private: + FilePlan file_; + std::shared_ptr result_; + }; + + /// Predicate results for all source files in one captured snapshot. + class EvaluatedPlan { + public: + EvaluatedPlan(int64_t snapshot_id, std::vector files) + : snapshot_id_(snapshot_id), files_(std::move(files)) {} + + int64_t SnapshotId() const { + return snapshot_id_; + } + + const std::vector& Files() const { + return files_; + } + + private: + int64_t snapshot_id_; + std::vector files_; + }; + + /// Creates a payload reader for one validated group. Returning a null reader marks the + /// index type as unusable so affected predicates keep their normal scan semantics. + using ReaderFactory = std::function>( + const FilePlan& file, const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group)>; + + /// Associates every active data file with the validated payload groups of its bucket. + /// + /// `index_entries` must be the ADD entries of the same snapshot carrying global index + /// metadata with source metadata. Splits must be non-streaming and their deletion + /// files, when present, must align with their data files. + static Result CreatePlan(int64_t snapshot_id, + const std::vector>& data_splits, + const std::vector& definitions, + const std::vector& index_entries); + + /// Evaluates the predicate for every planned file. Evaluation failures degrade to a + /// null per-file result instead of failing the scan. + static Result Evaluate(const Plan& plan, + const std::shared_ptr& table_schema, + const std::shared_ptr& predicate, + const std::vector& definitions, + const ReaderFactory& reader_factory); + + /// Creates the default reader factory which opens BTree payloads through the table's + /// index directory layout. Non-BTree families resolve to a null reader and therefore + /// keep normal scan semantics. + static ReaderFactory MakeReaderFactory( + const std::shared_ptr& file_system, + const std::shared_ptr& path_factories, + const std::shared_ptr& table_schema, + const std::map& options, const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader.h b/src/paimon/core/table/source/snapshot/snapshot_reader.h index c19299bb5..5e77cae86 100644 --- a/src/paimon/core/table/source/snapshot/snapshot_reader.h +++ b/src/paimon/core/table/source/snapshot/snapshot_reader.h @@ -92,6 +92,10 @@ class SnapshotReader { return scan_->GetSnapshotManager(); } + const std::unique_ptr& GetIndexFileHandler() const { + return index_file_handler_; + } + std::shared_ptr GetNonPartitionPredicate() const { return scan_->GetNonPartitionPredicate(); } diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 17add044b..722d57a30 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -33,6 +33,7 @@ #include "paimon/common/utils/options_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" #include "paimon/core/manifest/index_manifest_file.h" #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_list.h" @@ -51,6 +52,7 @@ #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" #include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/primary_key_index_batch_scan.h" #include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" #include "paimon/core/table/source/split_generator.h" @@ -299,12 +301,22 @@ Result> NewDataTableScan(const std::shared_ptr( /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit()); - if (!core_options.DataEvolutionEnabled()) { - return batch_scan; + if (core_options.DataEvolutionEnabled()) { + return std::make_unique( + context->GetPath(), snapshot_reader, std::move(batch_scan), + context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), + context->GetExecutor()); } - return std::make_unique( - context->GetPath(), snapshot_reader, std::move(batch_scan), context->GetGlobalIndexResult(), - core_options, context->GetMemoryPool(), context->GetExecutor()); + if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + if (!definitions.ScalarDefinitions().empty()) { + return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), + table_schema, path_factory, core_options, + context->GetMemoryPool()); + } + } + return batch_scan; } } // namespace From e90b5d65dea2e7352120e2d9c6c5994937b9ddd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=8B=87?= Date: Tue, 11 Aug 2026 02:26:25 -0400 Subject: [PATCH 03/14] feat(core): build source-backed primary-key BTree payloads Add PkSortedIndexFile::Build, which writes exactly one BTree payload for an ordered source group from value-sorted input and returns an IndexFileMeta carrying the serialized PrimaryKeyIndexSourceMeta, plus the end-to-end integration tests that exercise the full cycle with real BTree payloads: build -> plan -> evaluate -> localize -> splits, covering equality narrowing, ranges crossing file boundaries, empty hits omitting files, fallback on unindexed columns / uncovered files / corrupted metadata / out-of-range ordinals / over-fragmented results, deletion file alignment, and same-snapshot rejection. A user guide page documents the table requirements, semantics and current scope. close #192 --- docs/source/user_guide.rst | 1 + .../user_guide/primary_key_global_index.rst | 67 +++ src/paimon/CMakeLists.txt | 2 + .../index/pksorted/pk_sorted_index_file.cpp | 108 ++++ .../index/pksorted/pk_sorted_index_file.h | 69 +++ .../primary_key_sorted_index_scan_test.cpp | 551 ++++++++++++++++++ 6 files changed, 798 insertions(+) create mode 100644 docs/source/user_guide/primary_key_global_index.rst create mode 100644 src/paimon/core/index/pksorted/pk_sorted_index_file.cpp create mode 100644 src/paimon/core/index/pksorted/pk_sorted_index_file.h create mode 100644 src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index f0455343b..38ef0fc64 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -42,3 +42,4 @@ User Guide user_guide/prefetch user_guide/arrow user_guide/global_index + user_guide/primary_key_global_index diff --git a/docs/source/user_guide/primary_key_global_index.rst b/docs/source/user_guide/primary_key_global_index.rst new file mode 100644 index 000000000..61d543597 --- /dev/null +++ b/docs/source/user_guide/primary_key_global_index.rst @@ -0,0 +1,67 @@ +.. Copyright 2026-present Alibaba Inc. + +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +Primary Key Global Index +======================== + +Paimon 2.0 primary-key tables support *source-backed* global scalar indexes +(``pk-btree`` / ``pk-bitmap``). Unlike the Data Evolution global indexes described in +:doc:`global_index`, which address rows by a table-wide row id, a source-backed payload +covers the complete active source set of one positive data level of one bucket, and its +results are group ordinals that are localized back to per-file physical row positions. + +paimon-cpp supports the read path of this protocol: ordinary batch scans of a +primary-key table with scalar index definitions automatically evaluate the part of the +scan predicate that touches indexed fields against the validated payload groups of the +scanned snapshot, and narrow covered files to indexed splits carrying file-local row +ranges. No dedicated query API is required. + +Table requirements +------------------ + +The definitions follow the Java table options: + +- ``'pk-btree.index.columns' = 'price'`` with optional + ``'fields.price.pk-btree.index.options' = '{"block-size":"64 kb"}'`` +- fixed bucket (``bucket > 0``) or postpone bucket mode +- ``'deletion-vectors.enabled' = 'true'`` and ``'deletion-vectors.merge-on-read' = 'false'`` + +Semantics +--------- + +- A payload is only used when it provably covers the current active source set of its + data level: exactly one payload per level, source file names / order / row counts + identical to the active COMPACT files of that level, matching index type and field id, + and a row range of exactly ``[0, total source rows - 1]``. Anything else is treated as + uncovered and scanned normally. +- ``AND`` predicates narrow with any safely evaluable indexed child; ``OR`` predicates + only use the index when every branch is evaluable. Files whose evaluation fails, whose + positions are out of range, or whose result needs more than 4096 ranges fall back to a + normal scan individually. +- Indexed splits keep their deletion files aligned with the data file; the reader still + applies deletion vectors and the complete original predicate, so index results never + change visibility semantics. +- ``'global-index.enabled' = 'false'`` disables the planner. + +Current scope +------------- + +- The BTree payload reader is wired up. ``pk-bitmap`` (and vector / full-text) + definitions are recognized for validation, but their evaluation conservatively falls + back to a normal scan until their dedicated payload readers are supported. +- The read path targets the Java release-2.0.0 wire contract (source metadata v1, + ``GlobalIndexMeta`` with ``_SOURCE_META``, commit message v12). +- ``PkSortedIndexFile::Build`` can build one payload for an ordered source group from + value-sorted input, which supports tooling and tests; automatic build and maintenance + during compaction is not included yet. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 27455e98a..b481c3474 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -259,6 +259,7 @@ set(PAIMON_CORE_SRCS core/index/pk/primary_key_index_definitions.cpp core/index/pksorted/pk_sorted_index_group.cpp core/index/pksorted/pk_sorted_bucket_index_state.cpp + core/index/pksorted/pk_sorted_index_file.cpp core/io/generic_row_to_arrow_array_converter.cpp core/io/meta_to_arrow_array_converter.cpp core/io/async_key_value_producer_and_consumer.cpp @@ -859,6 +860,7 @@ if(PAIMON_BUILD_TESTS) core/table/sink/commit_message_test.cpp core/table/sink/commit_message_impl_test.cpp core/table/source/fallback_data_split_test.cpp + core/table/source/primary_key_sorted_index_scan_test.cpp core/table/source/table_read_test.cpp core/table/source/append_count_reader_test.cpp core/table/source/pk_count_reader_test.cpp diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp new file mode 100644 index 000000000..233f8d107 --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp @@ -0,0 +1,108 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/index/pksorted/pk_sorted_index_file.h" + +#include + +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/index/pk/primary_key_index_source_meta.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" + +namespace paimon { +Result> PkSortedIndexFile::Build( + const DataField& field, const std::string& index_type, + const std::map& options, int32_t data_level, + const std::vector& source_files, + const std::shared_ptr& sorted_values, const std::vector& sorted_ordinals, + const std::shared_ptr& file_writer, bool is_external_path, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexSourceMeta source_meta, + PrimaryKeyIndexSourceMeta::Create(data_level, source_files)); + int64_t source_row_count = 0; + for (const PrimaryKeyIndexSourceFile& source_file : source_files) { + if (__builtin_add_overflow(source_row_count, source_file.row_count, &source_row_count)) { + return Status::Invalid("Source row count overflows in sorted index build."); + } + } + if (source_row_count <= 0) { + return Status::Invalid("A sorted index group must reference at least one source row."); + } + if (sorted_values == nullptr || sorted_values->length() != source_row_count || + static_cast(sorted_ordinals.size()) != source_row_count) { + return Status::Invalid( + fmt::format("Sorted index input row count {} does not match source row count {}.", + sorted_values == nullptr ? 0 : sorted_values->length(), source_row_count)); + } + for (int64_t ordinal : sorted_ordinals) { + if (ordinal < 0 || ordinal >= source_row_count) { + return Status::Invalid( + fmt::format("Row id {} is outside sorted index group row range [0, {}).", ordinal, + source_row_count)); + } + } + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(index_type, options)); + if (indexer == nullptr) { + return Status::Invalid( + fmt::format("Unknown index type {}, may not registered", index_type)); + } + auto arrow_field = DataField::ConvertDataFieldToArrowField(field); + auto arrow_schema = arrow::schema({arrow_field}); + ArrowSchema c_arrow_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); + ScopeGuard schema_guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer, + indexer->CreateWriter(field.Name(), &c_arrow_schema, file_writer, pool)); + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr struct_array, + arrow::StructArray::Make({sorted_values}, {field.Name()})); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, &c_array)); + ScopeGuard array_guard([&]() { ArrowArrayRelease(&c_array); }); + std::vector ordinals = sorted_ordinals; + PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array, std::move(ordinals))); + PAIMON_ASSIGN_OR_RAISE(std::vector io_metas, writer->Finish()); + if (io_metas.size() != 1) { + return Status::Invalid(fmt::format( + "Sorted index build must produce exactly one payload file, but produced {}.", + io_metas.size())); + } + const GlobalIndexIOMeta& io_meta = io_metas[0]; + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr source_meta_bytes, + source_meta.Serialize(pool.get())); + std::optional external_path; + if (is_external_path) { + external_path = io_meta.file_path; + } + return std::make_shared( + index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, source_row_count, + /*dv_ranges=*/std::nullopt, external_path, + GlobalIndexMeta(0, source_row_count - 1, field.Id(), + /*extra_field_ids=*/std::nullopt, io_meta.metadata, source_meta_bytes)); +} + +} // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.h b/src/paimon/core/index/pksorted/pk_sorted_index_file.h new file mode 100644 index 000000000..49e6f74e2 --- /dev/null +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.h @@ -0,0 +1,69 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/pk/primary_key_index_source_file.h" +#include "paimon/global_index/io/global_index_file_writer.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { +/// Builds one source-backed primary-key index payload for an ordered set of physical data +/// files of a single data level. +/// +/// The caller provides all indexed values of the source group as one array sorted by +/// value, together with each value's zero-based ordinal in the ordered source group. The +/// builder writes exactly one payload file and returns its metadata carrying the +/// serialized `PrimaryKeyIndexSourceMeta`, so the payload can later be validated against +/// the active source set of its level. +class PkSortedIndexFile { + public: + PkSortedIndexFile() = delete; + ~PkSortedIndexFile() = delete; + + /// @param field The indexed field. + /// @param index_type The index algorithm identifier, e.g. "btree". + /// @param options The resolved index options (algorithm-prefixed keys included). + /// @param data_level The positive data level covered by the payload. + /// @param source_files The level's active source files ordered by file name. + /// @param sorted_values All indexed values of the source group sorted by value; nulls + /// may appear anywhere. + /// @param sorted_ordinals The group ordinal of each value, aligned with + /// `sorted_values`; every ordinal in `[0, total source rows)` must appear + /// exactly once. + /// @param file_writer The index-directory file writer of the payload's bucket. + /// @param is_external_path Whether `file_writer` resolves to an external index path. + static Result> Build( + const DataField& field, const std::string& index_type, + const std::map& options, int32_t data_level, + const std::vector& source_files, + const std::shared_ptr& sorted_values, + const std::vector& sorted_ordinals, + const std::shared_ptr& file_writer, bool is_external_path, + const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp new file mode 100644 index 000000000..77fb9f225 --- /dev/null +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -0,0 +1,551 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/source/primary_key_sorted_index_scan.h" + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/index/pksorted/pk_sorted_index_file.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/table/source/primary_key_sorted_index_result.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/global_index/io/global_index_file_writer.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { +constexpr int32_t kPriceFieldId = 1; +constexpr int64_t kSnapshotId = 7; +constexpr int64_t kFileARows = 100; +constexpr int64_t kFileBRows = 200; +constexpr int64_t kTotalRows = kFileARows + kFileBRows; + +class TestGlobalIndexFileWriter : public GlobalIndexFileWriter { + public: + TestGlobalIndexFileWriter(const std::shared_ptr& fs, const std::string& base_path) + : fs_(fs), base_path_(base_path) {} + + Result NewFileName(const std::string& prefix) const override { + return prefix + "-index-" + std::to_string(file_counter_++); + } + + Result> NewOutputStream( + const std::string& file_name) const override { + return fs_->Create(base_path_ + "/" + file_name, true); + } + + Result GetFileSize(const std::string& file_name) const override { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_status, + fs_->GetFileStatus(base_path_ + "/" + file_name)); + return file_status->GetLen(); + } + + std::string ToPath(const std::string& file_name) const override { + return base_path_ + "/" + file_name; + } + + private: + std::shared_ptr fs_; + std::string base_path_; + mutable int64_t file_counter_ = 0; +}; + +class TestGlobalIndexFileReader : public GlobalIndexFileReader { + public: + explicit TestGlobalIndexFileReader(const std::shared_ptr& fs) : fs_(fs) {} + + Result> GetInputStream( + const std::string& file_path) const override { + return fs_->Open(file_path); + } + + private: + std::shared_ptr fs_; +}; + +/// A reader stub whose equality result is fully controlled by the test, used to exercise +/// the untrusted-position fallbacks. +class StubGlobalIndexReader : public GlobalIndexReader { + public: + explicit StubGlobalIndexReader(RoaringBitmap64 equal_result) + : equal_result_(std::move(equal_result)) {} + + Result> VisitIsNotNull() override { + return NotEvaluable(); + } + Result> VisitIsNull() override { + return NotEvaluable(); + } + Result> VisitEqual(const Literal& literal) override { + RoaringBitmap64 copy = equal_result_; + return std::make_shared( + [bitmap = std::move(copy)]() -> Result { return bitmap; }); + } + Result> VisitNotEqual(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitLessThan(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitLessOrEqual(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitGreaterThan(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitGreaterOrEqual( + const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitIn( + const std::vector& literals) override { + return NotEvaluable(); + } + Result> VisitNotIn( + const std::vector& literals) override { + return NotEvaluable(); + } + Result> VisitStartsWith(const Literal& prefix) override { + return NotEvaluable(); + } + Result> VisitEndsWith(const Literal& suffix) override { + return NotEvaluable(); + } + Result> VisitContains(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitLike(const Literal& literal) override { + return NotEvaluable(); + } + Result> VisitVectorSearch( + const std::shared_ptr& vector_search) override { + return Status::Invalid("not supported"); + } + Result> VisitFullTextSearch( + const std::shared_ptr& full_text_search) override { + return Status::Invalid("not supported"); + } + bool IsThreadSafe() const override { + return false; + } + std::string GetIndexType() const override { + return "btree"; + } + + private: + static Result> NotEvaluable() { + return std::shared_ptr(nullptr); + } + + RoaringBitmap64 equal_result_; +}; +} // namespace + +class PrimaryKeySortedIndexScanTest : public ::testing::Test { + protected: + void SetUp() override { + pool_ = GetDefaultPool(); + test_dir_ = UniqueTestDirectory::Create("local"); + fs_ = test_dir_->GetFileSystem(); + base_path_ = test_dir_->Str(); + + std::vector fields = { + DataField(0, arrow::field("id", arrow::int64())), + DataField(kPriceFieldId, arrow::field("price", arrow::int64())), + DataField(2, arrow::field("status", arrow::utf8())), + }; + std::map options = {{"pk-btree.index.columns", "price"}}; + table_schema_ = std::make_shared( + /*version=*/3, /*id=*/0, fields, /*highest_field_id=*/2, + /*partition_keys=*/std::vector(), + /*primary_keys=*/std::vector{"id"}, options, + /*comment=*/std::nullopt, /*time_millis=*/0); + Result definitions = + PrimaryKeyIndexDefinitions::Create(*table_schema_); + ASSERT_OK(definitions.status()); + definitions_ = definitions.value().ScalarDefinitions(); + ASSERT_EQ(definitions_.size(), 1); + } + + std::shared_ptr MakeDataFile(const std::string& name, int64_t row_count, + int32_t level, const FileSource& file_source) { + return std::make_shared( + name, /*file_size=*/1024, row_count, + /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), + /*key_stats=*/SimpleStats::EmptyStats(), /*value_stats=*/SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/row_count, /*schema_id=*/0, level, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1721643142456LL, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, file_source, + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + } + + /// Builds the standard payload of this fixture: sources a.parquet(100) + b.parquet(200) + /// on level 5, indexed value at group ordinal `i` is `2 * i`. + Result> BuildPayload() { + std::vector source_files = {{"a.parquet", kFileARows}, + {"b.parquet", kFileBRows}}; + arrow::Int64Builder values_builder; + std::vector ordinals; + ordinals.reserve(kTotalRows); + for (int64_t i = 0; i < kTotalRows; i++) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(values_builder.Append(2 * i)); + ordinals.push_back(i); + } + std::shared_ptr sorted_values; + PAIMON_RETURN_NOT_OK_FROM_ARROW(values_builder.Finish(&sorted_values)); + PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema_->GetField(kPriceFieldId)); + auto file_writer = std::make_shared(fs_, base_path_); + return PkSortedIndexFile::Build(field, "btree", definitions_[0].Options(), + /*data_level=*/5, source_files, sorted_values, ordinals, + file_writer, /*is_external_path=*/false, pool_); + } + + std::shared_ptr MakeSplit( + const std::vector>& files, bool raw_convertible, + const std::vector>& deletion_files = {}) { + std::vector> data_files = files; + DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, + base_path_ + "/bucket-0", std::move(data_files)); + builder.WithSnapshot(kSnapshotId).IsStreaming(false).RawConvertible(raw_convertible); + if (!deletion_files.empty()) { + builder.WithDataDeletionFiles(deletion_files); + } + Result> split = builder.Build(); + assert(split.ok()); + return split.value(); + } + + std::vector MakeEntries(const std::shared_ptr& payload) { + return {IndexManifestEntry(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0, payload)}; + } + + PrimaryKeySortedIndexScan::ReaderFactory PayloadReaderFactory() { + std::shared_ptr fs = fs_; + std::string base_path = base_path_; + std::shared_ptr table_schema = table_schema_; + std::shared_ptr pool = pool_; + return [fs, base_path, table_schema, pool]( + const PrimaryKeySortedIndexScan::FilePlan& file, + const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr indexer, + GlobalIndexerFactory::Get(definition.IndexType(), definition.Options())); + if (indexer == nullptr) { + return Status::Invalid("btree indexer is not registered"); + } + const std::shared_ptr& payload = group.Payload(); + std::vector io_metas; + io_metas.emplace_back(base_path + "/" + payload->FileName(), payload->FileSize(), + payload->GetGlobalIndexMeta().value().index_meta); + PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(definition.FieldId())); + auto arrow_field = DataField::ConvertDataFieldToArrowField(field); + auto arrow_schema = arrow::schema({arrow_field}); + ArrowSchema c_arrow_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); + auto file_reader = std::make_shared(fs); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); + }; + } + + Result>> PlanEvaluateConvert( + const std::vector>& splits, + const std::vector& entries, const std::shared_ptr& predicate, + const PrimaryKeySortedIndexScan::ReaderFactory& reader_factory) { + PAIMON_ASSIGN_OR_RAISE( + PrimaryKeySortedIndexScan::Plan plan, + PrimaryKeySortedIndexScan::CreatePlan(kSnapshotId, splits, definitions_, entries)); + PAIMON_ASSIGN_OR_RAISE(PrimaryKeySortedIndexScan::EvaluatedPlan evaluated, + PrimaryKeySortedIndexScan::Evaluate(plan, table_schema_, predicate, + definitions_, reader_factory)); + return PrimaryKeySortedIndexResult::ToSplits(evaluated); + } + + std::shared_ptr PriceEqual(int64_t value) { + return PredicateBuilder::Equal(/*field_index=*/1, "price", FieldType::BIGINT, + Literal(value)); + } + + std::shared_ptr pool_; + std::shared_ptr test_dir_; + std::shared_ptr fs_; + std::string base_path_; + std::shared_ptr table_schema_; + std::vector definitions_; +}; + +TEST_F(PrimaryKeySortedIndexScanTest, EqualNarrowsToSingleFileRange) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + // Value 10 sits at group ordinal 5, i.e. row 5 of a.parquet. + Result>> splits = PlanEvaluateConvert( + {split}, MakeEntries(payload.value()), PriceEqual(10), PayloadReaderFactory()); + ASSERT_OK(splits.status()); + ASSERT_EQ(splits.value().size(), 1); + auto indexed_split = std::dynamic_pointer_cast(splits.value()[0]); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_EQ(inner_split->DataFiles().size(), 1); + ASSERT_EQ(inner_split->DataFiles()[0]->file_name, "a.parquet"); + ASSERT_EQ(indexed_split->RowRanges().size(), 1); + ASSERT_EQ(indexed_split->RowRanges()[0].from, 5); + ASSERT_EQ(indexed_split->RowRanges()[0].to, 5); +} + +TEST_F(PrimaryKeySortedIndexScanTest, RangeSpansFileBoundary) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + // Values in [190, 210] sit at group ordinals 95..105: rows 95..99 of a.parquet and + // rows 0..5 of b.parquet. + std::shared_ptr lower = PredicateBuilder::GreaterOrEqual( + /*field_index=*/1, "price", FieldType::BIGINT, Literal(static_cast(190))); + std::shared_ptr upper = PredicateBuilder::LessOrEqual( + /*field_index=*/1, "price", FieldType::BIGINT, Literal(static_cast(210))); + Result> predicate = PredicateBuilder::And({lower, upper}); + ASSERT_OK(predicate.status()); + Result>> splits = PlanEvaluateConvert( + {split}, MakeEntries(payload.value()), predicate.value(), PayloadReaderFactory()); + ASSERT_OK(splits.status()); + ASSERT_EQ(splits.value().size(), 2); + auto indexed_a = std::dynamic_pointer_cast(splits.value()[0]); + auto indexed_b = std::dynamic_pointer_cast(splits.value()[1]); + ASSERT_TRUE(indexed_a != nullptr); + ASSERT_TRUE(indexed_b != nullptr); + ASSERT_EQ(indexed_a->RowRanges().size(), 1); + ASSERT_EQ(indexed_a->RowRanges()[0].from, 95); + ASSERT_EQ(indexed_a->RowRanges()[0].to, 99); + ASSERT_EQ(indexed_b->RowRanges().size(), 1); + ASSERT_EQ(indexed_b->RowRanges()[0].from, 0); + ASSERT_EQ(indexed_b->RowRanges()[0].to, 5); +} + +TEST_F(PrimaryKeySortedIndexScanTest, EmptyResultOmitsAllFiles) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + // All indexed values are even, so 11 matches nothing. + Result>> splits = PlanEvaluateConvert( + {split}, MakeEntries(payload.value()), PriceEqual(11), PayloadReaderFactory()); + ASSERT_OK(splits.status()); + ASSERT_TRUE(splits.value().empty()); +} + +TEST_F(PrimaryKeySortedIndexScanTest, UnindexedFieldPredicateFallsBack) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/2, "status", FieldType::STRING, Literal(FieldType::STRING, "hit", 3)); + Result>> splits = PlanEvaluateConvert( + {split}, MakeEntries(payload.value()), predicate, PayloadReaderFactory()); + ASSERT_OK(splits.status()); + ASSERT_EQ(splits.value().size(), 2); + for (const std::shared_ptr& result_split : splits.value()) { + ASSERT_TRUE(std::dynamic_pointer_cast(result_split) == nullptr); + auto data_split = std::dynamic_pointer_cast(result_split); + ASSERT_TRUE(data_split != nullptr); + ASSERT_EQ(data_split->DataFiles().size(), 1); + } +} + +TEST_F(PrimaryKeySortedIndexScanTest, UncoveredFileFallsBackOthersNarrow) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact()), + MakeDataFile("c.parquet", 50, 0, FileSource::Append())}, + /*raw_convertible=*/true); + Result>> splits = PlanEvaluateConvert( + {split}, MakeEntries(payload.value()), PriceEqual(10), PayloadReaderFactory()); + ASSERT_OK(splits.status()); + // a.parquet narrows to an indexed split, b.parquet is omitted, c.parquet has no + // coverage and keeps a normal single-file scan. + ASSERT_EQ(splits.value().size(), 2); + auto indexed_split = std::dynamic_pointer_cast(splits.value()[0]); + ASSERT_TRUE(indexed_split != nullptr); + auto fallback_split = std::dynamic_pointer_cast(splits.value()[1]); + ASSERT_TRUE(fallback_split != nullptr); + ASSERT_EQ(fallback_split->DataFiles().size(), 1); + ASSERT_EQ(fallback_split->DataFiles()[0]->file_name, "c.parquet"); +} + +TEST_F(PrimaryKeySortedIndexScanTest, NonRawConvertibleSplitPreserved) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/false); + Result>> splits = PlanEvaluateConvert( + {split}, MakeEntries(payload.value()), PriceEqual(10), PayloadReaderFactory()); + ASSERT_OK(splits.status()); + ASSERT_EQ(splits.value().size(), 1); + ASSERT_EQ(splits.value()[0].get(), split.get()); +} + +TEST_F(PrimaryKeySortedIndexScanTest, InvalidRowRangePayloadFallsBack) { + Result> payload_result = BuildPayload(); + ASSERT_OK(payload_result.status()); + const std::shared_ptr& payload = payload_result.value(); + // Rebuild the payload metadata with a row range end beyond the source rows: the group + // validation must reject it and every file keeps a normal scan. + const GlobalIndexMeta& meta = payload->GetGlobalIndexMeta().value(); + auto broken_payload = std::make_shared( + payload->IndexType(), payload->FileName(), payload->FileSize(), payload->RowCount(), + std::nullopt, std::nullopt, + GlobalIndexMeta(meta.row_range_start, meta.row_range_end + 1, meta.index_field_id, + meta.extra_field_ids, meta.index_meta, meta.source_meta)); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + Result>> splits = PlanEvaluateConvert( + {split}, MakeEntries(broken_payload), PriceEqual(10), PayloadReaderFactory()); + ASSERT_OK(splits.status()); + ASSERT_EQ(splits.value().size(), 2); + for (const std::shared_ptr& result_split : splits.value()) { + ASSERT_TRUE(std::dynamic_pointer_cast(result_split) == nullptr); + } +} + +TEST_F(PrimaryKeySortedIndexScanTest, OutOfRangePositionsFailAllCoveredFiles) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + RoaringBitmap64 poisoned; + poisoned.Add(5); + poisoned.Add(kTotalRows + 10); + PrimaryKeySortedIndexScan::ReaderFactory stub_factory = + [&poisoned](const PrimaryKeySortedIndexScan::FilePlan& file, + const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + return std::make_shared(poisoned); + }; + Result>> splits = + PlanEvaluateConvert({split}, MakeEntries(payload.value()), PriceEqual(10), stub_factory); + ASSERT_OK(splits.status()); + // Both covered files must fall back to normal single-file scans. + ASSERT_EQ(splits.value().size(), 2); + for (const std::shared_ptr& result_split : splits.value()) { + ASSERT_TRUE(std::dynamic_pointer_cast(result_split) == nullptr); + } +} + +TEST_F(PrimaryKeySortedIndexScanTest, OverFragmentedResultFallsBack) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + // One data file, 20000 rows; every second row selected produces > 4096 ranges. + std::vector source_files = {{"big.parquet", 20000}}; + std::shared_ptr split = MakeSplit( + {MakeDataFile("big.parquet", 20000, 5, FileSource::Compact())}, /*raw_convertible=*/true); + Result> source_meta_bytes = [&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexSourceMeta source_meta, + PrimaryKeyIndexSourceMeta::Create(5, source_files)); + return source_meta.Serialize(pool_.get()); + }(); + ASSERT_OK(source_meta_bytes.status()); + auto big_payload = std::make_shared( + "btree", "big-index-file", /*file_size=*/1, /*row_count=*/20000, std::nullopt, std::nullopt, + GlobalIndexMeta(0, 19999, kPriceFieldId, std::nullopt, nullptr, source_meta_bytes.value())); + RoaringBitmap64 fragmented; + for (int64_t i = 0; i < 20000; i += 2) { + fragmented.Add(i); + } + PrimaryKeySortedIndexScan::ReaderFactory stub_factory = + [&fragmented]( + const PrimaryKeySortedIndexScan::FilePlan& file, + const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + return std::make_shared(fragmented); + }; + Result>> splits = + PlanEvaluateConvert({split}, MakeEntries(big_payload), PriceEqual(10), stub_factory); + ASSERT_OK(splits.status()); + ASSERT_EQ(splits.value().size(), 1); + ASSERT_TRUE(std::dynamic_pointer_cast(splits.value()[0]) == nullptr); +} + +TEST_F(PrimaryKeySortedIndexScanTest, DeletionFileStaysAlignedWithIndexedFile) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + DeletionFile deletion_file("dv-a", /*offset=*/0, /*length=*/16, /*cardinality=*/1); + std::shared_ptr split = MakeSplit( + {MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true, {std::optional(deletion_file), std::nullopt}); + Result>> splits = PlanEvaluateConvert( + {split}, MakeEntries(payload.value()), PriceEqual(10), PayloadReaderFactory()); + ASSERT_OK(splits.status()); + ASSERT_EQ(splits.value().size(), 1); + auto indexed_split = std::dynamic_pointer_cast(splits.value()[0]); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_EQ(inner_split->DeletionFiles().size(), 1); + ASSERT_TRUE(inner_split->DeletionFiles()[0] != std::nullopt); + ASSERT_EQ(inner_split->DeletionFiles()[0].value().path, "dv-a"); +} + +TEST_F(PrimaryKeySortedIndexScanTest, SnapshotMismatchIsRejected) { + Result> payload = BuildPayload(); + ASSERT_OK(payload.status()); + std::vector> files = { + MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact())}; + DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, base_path_, + std::move(files)); + builder.WithSnapshot(kSnapshotId + 1).IsStreaming(false).RawConvertible(true); + Result> split = builder.Build(); + ASSERT_OK(split.status()); + Result plan = PrimaryKeySortedIndexScan::CreatePlan( + kSnapshotId, {split.value()}, definitions_, MakeEntries(payload.value())); + ASSERT_NOK(plan.status()); +} + +} // namespace paimon::test From 4fa5817f3d8f495265b68981b5e67b25d49ee457 Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Tue, 11 Aug 2026 03:06:53 -0400 Subject: [PATCH 04/14] style: format primary-key index changes --- src/paimon/core/index/pk/primary_key_index_definitions.cpp | 4 ++-- src/paimon/core/index/pksorted/pk_sorted_index_file.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.cpp b/src/paimon/core/index/pk/primary_key_index_definitions.cpp index e23a51c62..3d3bd6efc 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions.cpp +++ b/src/paimon/core/index/pk/primary_key_index_definitions.cpp @@ -120,8 +120,8 @@ Result> SortedIndexOptions( } std::string key = member->name.GetString(); if (member->value.IsNull()) { - return Status::Invalid(fmt::format("{} value for key {} must not be null.", - option_key, key)); + return Status::Invalid( + fmt::format("{} value for key {} must not be null.", option_key, key)); } if (member->value.IsObject() || member->value.IsArray()) { return Status::Invalid( diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp index 233f8d107..2446866ac 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp @@ -22,8 +22,8 @@ #include "arrow/c/helpers.h" #include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/scope_guard.h" #include "paimon/core/index/pk/primary_key_index_source_meta.h" #include "paimon/global_index/global_index_io_meta.h" #include "paimon/global_index/global_index_writer.h" From a477bfbe2289584aa1968914f446bc88cc864c39 Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Tue, 11 Aug 2026 04:21:35 -0400 Subject: [PATCH 05/14] fix(pk-index): harden scalar index integration --- .../btree/btree_global_indexer.cpp | 8 ++- .../pk/primary_key_index_definitions.cpp | 58 +++++------------- .../index/pk/primary_key_index_definitions.h | 2 +- .../index/pk/primary_key_index_source_file.h | 4 +- .../pk/primary_key_index_source_meta.cpp | 11 +++- .../index/pk/primary_key_index_source_meta.h | 2 + .../pk/primary_key_index_source_meta_test.cpp | 6 ++ .../pk/primary_key_index_source_policy.h | 1 + .../pksorted/pk_sorted_bucket_index_state.cpp | 8 +++ .../pksorted/pk_sorted_bucket_index_state.h | 1 + .../pk_sorted_bucket_index_state_test.cpp | 15 +++++ .../index/pksorted/pk_sorted_index_file.cpp | 15 +++-- .../index/pksorted/pk_sorted_index_file.h | 5 +- .../index/pksorted/pk_sorted_index_group.h | 1 + .../core/operation/raw_file_split_read.cpp | 1 + .../core/operation/raw_file_split_read.h | 3 +- .../table/source/fallback_data_split_test.cpp | 37 ++++++++++++ .../core/table/source/fallback_table_read.cpp | 4 ++ .../source/primary_key_index_batch_scan.cpp | 2 +- .../primary_key_sorted_index_result.cpp | 22 +++++-- .../source/primary_key_sorted_index_scan.cpp | 10 ++-- .../source/primary_key_sorted_index_scan.h | 3 +- .../primary_key_sorted_index_scan_test.cpp | 60 +++++++++++-------- 23 files changed, 185 insertions(+), 94 deletions(-) diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index 29bffe9fe..f3e97b6ed 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -18,10 +18,12 @@ */ #include "paimon/common/global_index/btree/btree_global_indexer.h" +#include #include #include #include "arrow/c/bridge.h" +#include "fmt/format.h" #include "paimon/common/compression/block_compression_factory.h" #include "paimon/common/global_index/btree/btree_file_footer.h" #include "paimon/common/global_index/btree/btree_global_index_writer.h" @@ -40,6 +42,7 @@ #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/memory/bytes.h" #include "paimon/utils/roaring_bitmap64.h" + namespace paimon { Result> BTreeGlobalIndexer::Create( const std::map& options) { @@ -120,8 +123,9 @@ Result> BTreeGlobalIndexer::CreateReader( } read_buffer_size = static_cast(tmp_buffer_size); } - // TODO(lisizhuo.lsz): Allow users to specify an executor - std::shared_ptr executor = CreateDefaultExecutor(); + // Readers are created per payload group and may coexist for many buckets. Share the + // process-wide executor instead of creating a dedicated thread pool for every group. + std::shared_ptr executor = GetGlobalDefaultExecutor(); return std::make_shared(read_buffer_size, files, key_type, file_reader, cache_manager_, pool, executor); } diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.cpp b/src/paimon/core/index/pk/primary_key_index_definitions.cpp index 3d3bd6efc..98a378e95 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions.cpp +++ b/src/paimon/core/index/pk/primary_key_index_definitions.cpp @@ -20,6 +20,8 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/object_utils.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/defs.h" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" @@ -37,32 +39,15 @@ constexpr char kBitmapAlgorithmPrefix[] = "bitmap-index."; constexpr char kFieldScopedPrefix[] = "fields."; constexpr char kRecordsPerRangeKey[] = "sorted-index.records-per-range"; -std::string Trim(const std::string& value) { - size_t begin = value.find_first_not_of(" \t\r\n"); - if (begin == std::string::npos) { - return ""; - } - size_t end = value.find_last_not_of(" \t\r\n"); - return value.substr(begin, end - begin + 1); -} - std::vector IndexColumns(const std::map& options, const char* option_key) { auto iter = options.find(option_key); if (iter == options.end()) { return {}; } - std::vector columns; - const std::string& value = iter->second; - size_t start = 0; - while (true) { - size_t comma = value.find(',', start); - if (comma == std::string::npos) { - columns.push_back(Trim(value.substr(start))); - break; - } - columns.push_back(Trim(value.substr(start, comma - start))); - start = comma + 1; + std::vector columns = StringUtils::Split(iter->second, ",", false); + for (std::string& column : columns) { + StringUtils::Trim(&column); } return columns; } @@ -89,10 +74,6 @@ Status ValidateUniqueColumns(std::set* indexed_columns, return Status::OK(); } -bool StartsWith(const std::string& value, const char* prefix) { - return value.rfind(prefix, 0) == 0; -} - /// Resolves the effective option map of one sorted-index definition: table options first, /// then the field-scoped JSON options with unqualified keys prefixed by the algorithm /// prefix, mirroring Java `CoreOptions#primaryKeySortedIndexOptions`. @@ -104,7 +85,7 @@ Result> SortedIndexOptions( std::string option_key = fmt::format("{}{}.{}.index.options", kFieldScopedPrefix, column, option_family); auto iter = table_options.find(option_key); - if (iter == table_options.end() || Trim(iter->second).empty()) { + if (iter == table_options.end() || StringUtils::IsNullOrWhitespaceOnly(iter->second)) { return resolved; } @@ -115,7 +96,8 @@ Result> SortedIndexOptions( fmt::format("{} must be a JSON object of option key-value pairs.", option_key)); } for (auto member = document.MemberBegin(); member != document.MemberEnd(); ++member) { - if (!member->name.IsString() || Trim(member->name.GetString()).empty()) { + if (!member->name.IsString() || + StringUtils::IsNullOrWhitespaceOnly(member->name.GetString())) { return Status::Invalid(fmt::format("{} contains an empty option key.", option_key)); } std::string key = member->name.GetString(); @@ -138,10 +120,10 @@ Result> SortedIndexOptions( member->value.Accept(writer); value = buffer.GetString(); } - std::string qualified_key = - StartsWith(key, algorithm_prefix) || StartsWith(key, kFieldScopedPrefix) - ? key - : algorithm_prefix + key; + std::string qualified_key = StringUtils::StartsWith(key, algorithm_prefix) || + StringUtils::StartsWith(key, kFieldScopedPrefix) + ? key + : algorithm_prefix + key; auto previous = resolved.find(qualified_key); if (previous != resolved.end() && previous->second != value) { return Status::Invalid( @@ -152,14 +134,6 @@ Result> SortedIndexOptions( return resolved; } -bool Contains(const std::vector& columns, const std::string& column) { - for (const std::string& candidate : columns) { - if (candidate == column) { - return true; - } - } - return false; -} } // namespace Result PrimaryKeyIndexDefinitions::Create(const TableSchema& schema) { @@ -185,21 +159,21 @@ Result PrimaryKeyIndexDefinitions::Create(const Tabl std::vector definitions; for (const DataField& field : schema.Fields()) { const std::string& column = field.Name(); - if (Contains(btree_columns, column)) { + if (ObjectUtils::Contains(btree_columns, column)) { Result> definition_options = SortedIndexOptions(options, column, kBTreeOptionFamily, kBTreeAlgorithmPrefix); PAIMON_RETURN_NOT_OK(definition_options.status()); definitions.emplace_back(column, field.Id(), kBTreeIndexType, std::move(definition_options).value(), PrimaryKeyIndexDefinition::Family::BTREE); - } else if (Contains(bitmap_columns, column)) { + } else if (ObjectUtils::Contains(bitmap_columns, column)) { Result> definition_options = SortedIndexOptions(options, column, kBitmapOptionFamily, kBitmapAlgorithmPrefix); PAIMON_RETURN_NOT_OK(definition_options.status()); definitions.emplace_back(column, field.Id(), kBitmapIndexType, std::move(definition_options).value(), PrimaryKeyIndexDefinition::Family::BITMAP); - } else if (Contains(vector_columns, column)) { + } else if (ObjectUtils::Contains(vector_columns, column)) { std::string index_type; auto type_iter = options.find(fmt::format("{}{}.pk-vector.index.type", kFieldScopedPrefix, column)); @@ -209,7 +183,7 @@ Result PrimaryKeyIndexDefinitions::Create(const Tabl definitions.emplace_back(column, field.Id(), index_type, std::map(), PrimaryKeyIndexDefinition::Family::VECTOR); - } else if (Contains(full_text_columns, column)) { + } else if (ObjectUtils::Contains(full_text_columns, column)) { definitions.emplace_back(column, field.Id(), kFullTextIndexType, std::map(), PrimaryKeyIndexDefinition::Family::FULL_TEXT); diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.h b/src/paimon/core/index/pk/primary_key_index_definitions.h index c85fdd629..d9a5324b3 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions.h +++ b/src/paimon/core/index/pk/primary_key_index_definitions.h @@ -17,8 +17,8 @@ #pragma once #include -#include #include +#include #include #include "paimon/core/index/pk/primary_key_index_definition.h" diff --git a/src/paimon/core/index/pk/primary_key_index_source_file.h b/src/paimon/core/index/pk/primary_key_index_source_file.h index 23d9c303c..3575f43f6 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_file.h +++ b/src/paimon/core/index/pk/primary_key_index_source_file.h @@ -26,8 +26,8 @@ namespace paimon { /// Two source files are interchangeable only when both the file name and the row count /// match; coverage validation relies on this strict identity. struct PrimaryKeyIndexSourceFile { - PrimaryKeyIndexSourceFile(std::string _file_name, int64_t _row_count) - : file_name(std::move(_file_name)), row_count(_row_count) {} + PrimaryKeyIndexSourceFile(std::string file_name, int64_t row_count) + : file_name(std::move(file_name)), row_count(row_count) {} bool operator==(const PrimaryKeyIndexSourceFile& other) const { return file_name == other.file_name && row_count == other.row_count; diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp index 646b50881..ddfc89f68 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp @@ -16,6 +16,7 @@ #include "paimon/core/index/pk/primary_key_index_source_meta.h" +#include #include #include #include @@ -30,6 +31,7 @@ namespace { // Each serialized entry needs at least the two-byte writeUTF length and one int64 row count, // mirroring the defensive source file count cap of the Java deserializer. constexpr size_t kMinBytesPerSourceFile = sizeof(uint16_t) + sizeof(int64_t); +constexpr size_t kMaxInitialSourceFileCapacity = 1024; void AppendBigEndian32(int32_t value, std::string* out) { uint32_t bits = static_cast(value); @@ -114,6 +116,12 @@ Result PrimaryKeyIndexSourceMeta::Create( if (source_files.empty()) { return Status::Invalid("An index must reference source files."); } + for (const PrimaryKeyIndexSourceFile& source_file : source_files) { + if (source_file.row_count < 0) { + return Status::Invalid(fmt::format("Source file {} has a negative row count {}.", + source_file.file_name, source_file.row_count)); + } + } return PrimaryKeyIndexSourceMeta(data_level, std::move(source_files)); } @@ -148,7 +156,8 @@ Result PrimaryKeyIndexSourceMeta::Deserialize(const c source_file_count, maximum_source_file_count)); } std::vector source_files; - source_files.reserve(source_file_count); + source_files.reserve( + std::min(static_cast(source_file_count), kMaxInitialSourceFileCapacity)); for (int32_t i = 0; i < source_file_count; i++) { PAIMON_ASSIGN_OR_RAISE(uint16_t name_length, cursor.ReadUint16()); PAIMON_ASSIGN_OR_RAISE(std::string_view name_bytes, cursor.ReadBytes(name_length)); diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.h b/src/paimon/core/index/pk/primary_key_index_source_meta.h index 97ce35407..58f1950ca 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta.h +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.h @@ -16,9 +16,11 @@ #pragma once +#include #include #include #include +#include #include #include "paimon/core/index/pk/primary_key_index_source_file.h" diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp index 2311d6861..c9e4fcc2b 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp +++ b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp @@ -142,6 +142,11 @@ TEST_F(PrimaryKeyIndexSourceMetaTest, DeserializeRejectsBadPayloads) { // Buffer cut in the middle of the row count: only 4 of the 8 bytes remain. std::string cut_row_count = valid.substr(0, 27); ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(cut_row_count.data(), cut_row_count.size())); + + std::string negative_row_count = valid; + negative_row_count[23] = '\xFF'; + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Deserialize(negative_row_count.data(), + negative_row_count.size())); } TEST_F(PrimaryKeyIndexSourceMetaTest, CreateRejectsInvalidArguments) { @@ -150,6 +155,7 @@ TEST_F(PrimaryKeyIndexSourceMetaTest, CreateRejectsInvalidArguments) { ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(0, files)); ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(-1, files)); ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(3, {})); + ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(3, {{"a.parquet", -1}})); } TEST_F(PrimaryKeyIndexSourceMetaTest, FromIndexFileDecodesSourceMeta) { diff --git a/src/paimon/core/index/pk/primary_key_index_source_policy.h b/src/paimon/core/index/pk/primary_key_index_source_policy.h index b6c366461..410461e8d 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_policy.h +++ b/src/paimon/core/index/pk/primary_key_index_source_policy.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp index e86bf0956..d9fbf2026 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp @@ -18,9 +18,11 @@ #include #include +#include #include #include +#include "paimon/core/index/global_index_meta.h" #include "paimon/core/index/pk/primary_key_index_source_meta.h" #include "paimon/core/index/pk/primary_key_index_source_policy.h" @@ -53,6 +55,12 @@ PkSortedBucketIndexState PkSortedBucketIndexState::FromActiveDataFiles( if (payload == nullptr) { continue; } + const std::optional& global_index_meta = payload->GetGlobalIndexMeta(); + if (payload->IndexType() != index_type || global_index_meta == std::nullopt || + global_index_meta->index_field_id != field_id) { + rejected.push_back(payload); + continue; + } Result source_meta_result = PrimaryKeyIndexSourceMeta::FromIndexFile(*payload); if (!source_meta_result.ok()) { diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h index 2852d61d0..fa4ee8f5e 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "paimon/core/index/index_file_meta.h" diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp index d6198d778..25dbaa83b 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp @@ -209,6 +209,21 @@ TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongIndexType) { ASSERT_EQ(1, state.UncoveredSourceFiles().size()); } +TEST_F(PkSortedBucketIndexStateTest, WrongCandidateDoesNotMaskValidPayload) { + std::vector> data_files = { + MakeDataFile("a", 100, 5, FileSource::Compact())}; + std::shared_ptr valid_payload = MakePayload(7, "btree", 5, {{"a", 100}}, 100); + std::shared_ptr wrong_payload = + MakePayload(/*field_id=*/8, "btree", 5, {{"a", 100}}, 100); + PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( + 7, "btree", data_files, {valid_payload, wrong_payload}); + ASSERT_EQ(1, state.Groups().size()); + ASSERT_EQ(valid_payload, state.Groups()[0].Payload()); + ASSERT_EQ(1, state.RejectedPayloads().size()); + ASSERT_EQ(wrong_payload, state.RejectedPayloads()[0]); + ASSERT_TRUE(state.UncoveredSourceFiles().empty()); +} + TEST_F(PkSortedBucketIndexStateTest, RejectsPayloadWithWrongRowRange) { std::vector> data_files = { MakeDataFile("a", 100, 5, FileSource::Compact()), diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp index 2446866ac..4a428b82e 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp @@ -16,6 +16,8 @@ #include "paimon/core/index/pksorted/pk_sorted_index_file.h" +#include +#include #include #include "arrow/c/bridge.h" @@ -35,7 +37,7 @@ Result> PkSortedIndexFile::Build( const DataField& field, const std::string& index_type, const std::map& options, int32_t data_level, const std::vector& source_files, - const std::shared_ptr& sorted_values, const std::vector& sorted_ordinals, + const std::shared_ptr& sorted_values, std::vector sorted_ordinals, const std::shared_ptr& file_writer, bool is_external_path, const std::shared_ptr& pool) { PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexSourceMeta source_meta, @@ -55,19 +57,23 @@ Result> PkSortedIndexFile::Build( fmt::format("Sorted index input row count {} does not match source row count {}.", sorted_values == nullptr ? 0 : sorted_values->length(), source_row_count)); } + std::vector seen_ordinals(static_cast(source_row_count), false); for (int64_t ordinal : sorted_ordinals) { if (ordinal < 0 || ordinal >= source_row_count) { return Status::Invalid( fmt::format("Row id {} is outside sorted index group row range [0, {}).", ordinal, source_row_count)); } + if (seen_ordinals[ordinal]) { + return Status::Invalid(fmt::format("Row id {} appears more than once.", ordinal)); + } + seen_ordinals[ordinal] = true; } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, GlobalIndexerFactory::Get(index_type, options)); if (indexer == nullptr) { - return Status::Invalid( - fmt::format("Unknown index type {}, may not registered", index_type)); + return Status::Invalid(fmt::format("Index type {} is not registered.", index_type)); } auto arrow_field = DataField::ConvertDataFieldToArrowField(field); auto arrow_schema = arrow::schema({arrow_field}); @@ -82,8 +88,7 @@ Result> PkSortedIndexFile::Build( ::ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, &c_array)); ScopeGuard array_guard([&]() { ArrowArrayRelease(&c_array); }); - std::vector ordinals = sorted_ordinals; - PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array, std::move(ordinals))); + PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array, std::move(sorted_ordinals))); PAIMON_ASSIGN_OR_RAISE(std::vector io_metas, writer->Finish()); if (io_metas.size() != 1) { return Status::Invalid(fmt::format( diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.h b/src/paimon/core/index/pksorted/pk_sorted_index_file.h index 49e6f74e2..7f350314f 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_file.h +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.h @@ -56,12 +56,13 @@ class PkSortedIndexFile { /// exactly once. /// @param file_writer The index-directory file writer of the payload's bucket. /// @param is_external_path Whether `file_writer` resolves to an external index path. + /// @param pool The memory pool used for metadata and index construction. + /// @return Metadata for the single payload file written by the index builder. static Result> Build( const DataField& field, const std::string& index_type, const std::map& options, int32_t data_level, const std::vector& source_files, - const std::shared_ptr& sorted_values, - const std::vector& sorted_ordinals, + const std::shared_ptr& sorted_values, std::vector sorted_ordinals, const std::shared_ptr& file_writer, bool is_external_path, const std::shared_ptr& pool); }; diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_group.h b/src/paimon/core/index/pksorted/pk_sorted_index_group.h index b9cb85cff..01a2a67cd 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_group.h +++ b/src/paimon/core/index/pksorted/pk_sorted_index_group.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "paimon/core/index/index_file_meta.h" diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index 4d14b537d..8bb59987e 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -25,6 +25,7 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" +#include "fmt/format.h" #include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h" #include "paimon/common/reader/complete_row_kind_batch_reader.h" #include "paimon/common/reader/concat_batch_reader.h" diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 35fa6576a..9b87dc835 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -19,8 +19,7 @@ #pragma once #include -#include -#include +#include #include #include "paimon/core/core_options.h" diff --git a/src/paimon/core/table/source/fallback_data_split_test.cpp b/src/paimon/core/table/source/fallback_data_split_test.cpp index 18fed0cf7..80d290560 100644 --- a/src/paimon/core/table/source/fallback_data_split_test.cpp +++ b/src/paimon/core/table/source/fallback_data_split_test.cpp @@ -27,10 +27,12 @@ #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" +#include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/fallback_table_read.h" #include "paimon/data/timestamp.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" @@ -41,6 +43,41 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { +class TrackingTableRead : public TableRead { + public: + explicit TrackingTableRead(const std::shared_ptr& pool) : TableRead(pool) {} + + Result> CreateReader( + const std::shared_ptr& split) override { + last_split_ = split; + return std::unique_ptr(); + } + + std::shared_ptr last_split_; +}; +} // namespace + +TEST(FallbackTableReadTest, RoutesIndexedSplitToMainTable) { + std::shared_ptr pool = GetDefaultPool(); + auto main_table = std::make_unique(pool); + auto fallback_table = std::make_unique(pool); + TrackingTableRead* main_table_ptr = main_table.get(); + TrackingTableRead* fallback_table_ptr = fallback_table.get(); + FallbackTableRead table_read(std::move(main_table), std::move(fallback_table), pool); + + DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, /*bucket_path=*/"", + /*data_files=*/{}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr data_split, + builder.IsStreaming(false).RawConvertible(true).Build()); + std::shared_ptr indexed_split = + std::make_shared(data_split, std::vector()); + + ASSERT_OK(table_read.CreateReader(indexed_split)); + ASSERT_EQ(indexed_split, main_table_ptr->last_split_); + ASSERT_EQ(nullptr, fallback_table_ptr->last_split_); +} + TEST(FallbackDataSplitTest, TestDeserialize) { std::string file_name = paimon::test::GetDataDir() + "/parquet/append_table_with_append_pt_branch.db/" diff --git a/src/paimon/core/table/source/fallback_table_read.cpp b/src/paimon/core/table/source/fallback_table_read.cpp index 6ca44215e..5f5957609 100644 --- a/src/paimon/core/table/source/fallback_table_read.cpp +++ b/src/paimon/core/table/source/fallback_table_read.cpp @@ -21,6 +21,7 @@ #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/global_index/indexed_split.h" #include "paimon/status.h" #include "paimon/table/source/data_split.h" @@ -35,6 +36,9 @@ Result> FallbackTableRead::CreateReader( return main_table_->CreateReader(fallback_data_split->GetSplit()); } } + if (std::dynamic_pointer_cast(split) != nullptr) { + return main_table_->CreateReader(split); + } auto data_split = std::dynamic_pointer_cast(split); if (!data_split) { return Status::Invalid("cannot cast split to data split"); diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp index b5dbb153a..4b5c48b74 100644 --- a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp @@ -252,7 +252,7 @@ Result> PrimaryKeyIndexBatchScan::CreatePlan() { PrimaryKeySortedIndexScan::ReaderFactory reader_factory = PrimaryKeySortedIndexScan::MakeReaderFactory( core_options_.GetFileSystem(), std::make_shared(path_factory_), - table_schema_, core_options_.ToMap(), pool_); + table_schema_, pool_); PAIMON_ASSIGN_OR_RAISE( PrimaryKeySortedIndexScan::EvaluatedPlan evaluated_plan, PrimaryKeySortedIndexScan::Evaluate(index_plan, table_schema_, index_predicate, diff --git a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp index c0ebfadb4..4e77f5c7d 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp @@ -17,6 +17,7 @@ #include "paimon/core/table/source/primary_key_sorted_index_result.h" #include +#include #include #include #include @@ -81,14 +82,27 @@ Result>> ToRanges(const GlobalIndexResult& resu Result>> PrimaryKeySortedIndexResult::ToSplits( const PrimaryKeySortedIndexScan::EvaluatedPlan& evaluated_plan) { + std::map preserve_raw_splits; + for (const PrimaryKeySortedIndexScan::EvaluatedFile& evaluated_file : evaluated_plan.Files()) { + const std::shared_ptr& source_split = evaluated_file.File().SourceSplit(); + if (!source_split->RawConvertible()) { + continue; + } + auto iter = preserve_raw_splits.emplace(source_split.get(), true).first; + if (evaluated_file.IndexResult() != nullptr) { + iter->second = false; + } + } + std::vector> splits; - std::set preserved_non_raw_splits; + std::set preserved_splits; for (const PrimaryKeySortedIndexScan::EvaluatedFile& evaluated_file : evaluated_plan.Files()) { const PrimaryKeySortedIndexScan::FilePlan& file = evaluated_file.File(); const std::shared_ptr& source_split = file.SourceSplit(); - if (!source_split->RawConvertible()) { - // Splits that cannot be read file by file keep their original shape. - if (preserved_non_raw_splits.insert(source_split.get()).second) { + if (!source_split->RawConvertible() || preserve_raw_splits[source_split.get()]) { + // Preserve the planner's bin packing when the split cannot be read file by file + // or no file in the split has a usable index result. + if (preserved_splits.insert(source_split.get()).second) { splits.push_back(source_split); } continue; diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp index fe7689c3e..328c69110 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -521,10 +521,9 @@ class FsGlobalIndexFileReader : public GlobalIndexFileReader { PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFactory( const std::shared_ptr& file_system, const std::shared_ptr& path_factories, - const std::shared_ptr& table_schema, - const std::map& options, const std::shared_ptr& pool) { + const std::shared_ptr& table_schema, const std::shared_ptr& pool) { auto file_reader = std::make_shared(file_system); - return [path_factories, table_schema, options, pool, file_reader]( + return [path_factories, table_schema, pool, file_reader]( const FilePlan& file, const PrimaryKeyIndexDefinition& definition, const PkSortedIndexGroup& group) -> Result> { if (definition.GetFamily() != PrimaryKeyIndexDefinition::Family::BTREE) { @@ -532,8 +531,9 @@ PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFa // semantics until their dedicated readers are supported. return std::shared_ptr(nullptr); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, - GlobalIndexerFactory::Get(definition.IndexType(), options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr indexer, + GlobalIndexerFactory::Get(definition.IndexType(), definition.Options())); if (indexer == nullptr) { return std::shared_ptr(nullptr); } diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.h b/src/paimon/core/table/source/primary_key_sorted_index_scan.h index c59bf8ed6..64e3fb038 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.h +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.h @@ -176,8 +176,7 @@ class PrimaryKeySortedIndexScan { static ReaderFactory MakeReaderFactory( const std::shared_ptr& file_system, const std::shared_ptr& path_factories, - const std::shared_ptr& table_schema, - const std::map& options, const std::shared_ptr& pool); + const std::shared_ptr& table_schema, const std::shared_ptr& pool); }; } // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index 77fb9f225..be1552a2c 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -22,6 +22,7 @@ #include #include "arrow/api.h" +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" @@ -52,7 +53,7 @@ class TestGlobalIndexFileWriter : public GlobalIndexFileWriter { : fs_(fs), base_path_(base_path) {} Result NewFileName(const std::string& prefix) const override { - return prefix + "-index-" + std::to_string(file_counter_++); + return fmt::format("{}-index-{}", prefix, file_counter_++); } Result> NewOutputStream( @@ -207,25 +208,32 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); } - /// Builds the standard payload of this fixture: sources a.parquet(100) + b.parquet(200) - /// on level 5, indexed value at group ordinal `i` is `2 * i`. - Result> BuildPayload() { + Result> BuildPayload(std::vector ordinals) { std::vector source_files = {{"a.parquet", kFileARows}, {"b.parquet", kFileBRows}}; arrow::Int64Builder values_builder; - std::vector ordinals; - ordinals.reserve(kTotalRows); for (int64_t i = 0; i < kTotalRows; i++) { PAIMON_RETURN_NOT_OK_FROM_ARROW(values_builder.Append(2 * i)); - ordinals.push_back(i); } std::shared_ptr sorted_values; PAIMON_RETURN_NOT_OK_FROM_ARROW(values_builder.Finish(&sorted_values)); PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema_->GetField(kPriceFieldId)); auto file_writer = std::make_shared(fs_, base_path_); return PkSortedIndexFile::Build(field, "btree", definitions_[0].Options(), - /*data_level=*/5, source_files, sorted_values, ordinals, - file_writer, /*is_external_path=*/false, pool_); + /*data_level=*/5, source_files, sorted_values, + std::move(ordinals), file_writer, + /*is_external_path=*/false, pool_); + } + + /// Builds the standard payload of this fixture: sources a.parquet(100) + b.parquet(200) + /// on level 5, indexed value at group ordinal `i` is `2 * i`. + Result> BuildPayload() { + std::vector ordinals; + ordinals.reserve(kTotalRows); + for (int64_t i = 0; i < kTotalRows; i++) { + ordinals.push_back(i); + } + return BuildPayload(std::move(ordinals)); } std::shared_ptr MakeSplit( @@ -325,6 +333,17 @@ TEST_F(PrimaryKeySortedIndexScanTest, EqualNarrowsToSingleFileRange) { ASSERT_EQ(indexed_split->RowRanges()[0].to, 5); } +TEST_F(PrimaryKeySortedIndexScanTest, BuildRejectsDuplicateOrdinals) { + std::vector ordinals; + ordinals.reserve(kTotalRows); + for (int64_t i = 0; i < kTotalRows; i++) { + ordinals.push_back(i); + } + ordinals[1] = 0; + ASSERT_NOK_WITH_MSG(BuildPayload(std::move(ordinals)).status(), + "Row id 0 appears more than once"); +} + TEST_F(PrimaryKeySortedIndexScanTest, RangeSpansFileBoundary) { Result> payload = BuildPayload(); ASSERT_OK(payload.status()); @@ -382,13 +401,8 @@ TEST_F(PrimaryKeySortedIndexScanTest, UnindexedFieldPredicateFallsBack) { Result>> splits = PlanEvaluateConvert( {split}, MakeEntries(payload.value()), predicate, PayloadReaderFactory()); ASSERT_OK(splits.status()); - ASSERT_EQ(splits.value().size(), 2); - for (const std::shared_ptr& result_split : splits.value()) { - ASSERT_TRUE(std::dynamic_pointer_cast(result_split) == nullptr); - auto data_split = std::dynamic_pointer_cast(result_split); - ASSERT_TRUE(data_split != nullptr); - ASSERT_EQ(data_split->DataFiles().size(), 1); - } + ASSERT_EQ(1, splits.value().size()); + ASSERT_EQ(split, splits.value()[0]); } TEST_F(PrimaryKeySortedIndexScanTest, UncoveredFileFallsBackOthersNarrow) { @@ -446,10 +460,8 @@ TEST_F(PrimaryKeySortedIndexScanTest, InvalidRowRangePayloadFallsBack) { Result>> splits = PlanEvaluateConvert( {split}, MakeEntries(broken_payload), PriceEqual(10), PayloadReaderFactory()); ASSERT_OK(splits.status()); - ASSERT_EQ(splits.value().size(), 2); - for (const std::shared_ptr& result_split : splits.value()) { - ASSERT_TRUE(std::dynamic_pointer_cast(result_split) == nullptr); - } + ASSERT_EQ(1, splits.value().size()); + ASSERT_EQ(split, splits.value()[0]); } TEST_F(PrimaryKeySortedIndexScanTest, OutOfRangePositionsFailAllCoveredFiles) { @@ -471,11 +483,9 @@ TEST_F(PrimaryKeySortedIndexScanTest, OutOfRangePositionsFailAllCoveredFiles) { Result>> splits = PlanEvaluateConvert({split}, MakeEntries(payload.value()), PriceEqual(10), stub_factory); ASSERT_OK(splits.status()); - // Both covered files must fall back to normal single-file scans. - ASSERT_EQ(splits.value().size(), 2); - for (const std::shared_ptr& result_split : splits.value()) { - ASSERT_TRUE(std::dynamic_pointer_cast(result_split) == nullptr); - } + // Both covered files fall back together, preserving the planner's original bin packing. + ASSERT_EQ(1, splits.value().size()); + ASSERT_EQ(split, splits.value()[0]); } TEST_F(PrimaryKeySortedIndexScanTest, OverFragmentedResultFallsBack) { From d5990c97b3916cb0f55a10021a4882c096420fbf Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Tue, 11 Aug 2026 05:20:14 -0400 Subject: [PATCH 06/14] chore: normalize PK index license headers --- .../user_guide/primary_key_global_index.rst | 23 +++++++++++-------- .../common/utils/java_modified_utf8.cpp | 23 +++++++++++-------- src/paimon/common/utils/java_modified_utf8.h | 23 +++++++++++-------- .../common/utils/java_modified_utf8_test.cpp | 23 +++++++++++-------- .../index/pk/primary_key_index_definition.h | 23 +++++++++++-------- .../pk/primary_key_index_definitions.cpp | 23 +++++++++++-------- .../index/pk/primary_key_index_definitions.h | 23 +++++++++++-------- .../pk/primary_key_index_definitions_test.cpp | 23 +++++++++++-------- .../index/pk/primary_key_index_source_file.h | 23 +++++++++++-------- .../pk/primary_key_index_source_meta.cpp | 23 +++++++++++-------- .../index/pk/primary_key_index_source_meta.h | 23 +++++++++++-------- .../pk/primary_key_index_source_meta_test.cpp | 23 +++++++++++-------- .../pk/primary_key_index_source_policy.h | 23 +++++++++++-------- .../pksorted/pk_sorted_bucket_index_state.cpp | 23 +++++++++++-------- .../pksorted/pk_sorted_bucket_index_state.h | 23 +++++++++++-------- .../pk_sorted_bucket_index_state_test.cpp | 23 +++++++++++-------- .../index/pksorted/pk_sorted_index_file.cpp | 23 +++++++++++-------- .../index/pksorted/pk_sorted_index_file.h | 23 +++++++++++-------- .../index/pksorted/pk_sorted_index_group.cpp | 23 +++++++++++-------- .../index/pksorted/pk_sorted_index_group.h | 23 +++++++++++-------- .../source/primary_key_index_batch_scan.cpp | 23 +++++++++++-------- .../source/primary_key_index_batch_scan.h | 23 +++++++++++-------- .../primary_key_sorted_index_result.cpp | 23 +++++++++++-------- .../source/primary_key_sorted_index_result.h | 23 +++++++++++-------- .../source/primary_key_sorted_index_scan.cpp | 23 +++++++++++-------- .../source/primary_key_sorted_index_scan.h | 23 +++++++++++-------- .../primary_key_sorted_index_scan_test.cpp | 23 +++++++++++-------- 27 files changed, 351 insertions(+), 270 deletions(-) diff --git a/docs/source/user_guide/primary_key_global_index.rst b/docs/source/user_guide/primary_key_global_index.rst index 61d543597..b820dcdce 100644 --- a/docs/source/user_guide/primary_key_global_index.rst +++ b/docs/source/user_guide/primary_key_global_index.rst @@ -1,16 +1,19 @@ -.. Copyright 2026-present Alibaba Inc. - -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at .. http://www.apache.org/licenses/LICENSE-2.0 -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. Primary Key Global Index ======================== diff --git a/src/paimon/common/utils/java_modified_utf8.cpp b/src/paimon/common/utils/java_modified_utf8.cpp index 1303848fa..554cd1637 100644 --- a/src/paimon/common/utils/java_modified_utf8.cpp +++ b/src/paimon/common/utils/java_modified_utf8.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/common/utils/java_modified_utf8.h" diff --git a/src/paimon/common/utils/java_modified_utf8.h b/src/paimon/common/utils/java_modified_utf8.h index c5fbf5abe..84dbfe537 100644 --- a/src/paimon/common/utils/java_modified_utf8.h +++ b/src/paimon/common/utils/java_modified_utf8.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/common/utils/java_modified_utf8_test.cpp b/src/paimon/common/utils/java_modified_utf8_test.cpp index 576c2e081..6eaecc5b5 100644 --- a/src/paimon/common/utils/java_modified_utf8_test.cpp +++ b/src/paimon/common/utils/java_modified_utf8_test.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/common/utils/java_modified_utf8.h" diff --git a/src/paimon/core/index/pk/primary_key_index_definition.h b/src/paimon/core/index/pk/primary_key_index_definition.h index 3506877e8..098903e1c 100644 --- a/src/paimon/core/index/pk/primary_key_index_definition.h +++ b/src/paimon/core/index/pk/primary_key_index_definition.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.cpp b/src/paimon/core/index/pk/primary_key_index_definitions.cpp index 98a378e95..57fd45ae0 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions.cpp +++ b/src/paimon/core/index/pk/primary_key_index_definitions.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/index/pk/primary_key_index_definitions.h" diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.h b/src/paimon/core/index/pk/primary_key_index_definitions.h index d9a5324b3..37f20da5f 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions.h +++ b/src/paimon/core/index/pk/primary_key_index_definitions.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp b/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp index 86595b0e8..8649e26d8 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp +++ b/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/index/pk/primary_key_index_definitions.h" diff --git a/src/paimon/core/index/pk/primary_key_index_source_file.h b/src/paimon/core/index/pk/primary_key_index_source_file.h index 3575f43f6..c1afa3de7 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_file.h +++ b/src/paimon/core/index/pk/primary_key_index_source_file.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp index ddfc89f68..f70b8f78a 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/index/pk/primary_key_index_source_meta.h" diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.h b/src/paimon/core/index/pk/primary_key_index_source_meta.h index 58f1950ca..639f59852 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta.h +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp index c9e4fcc2b..478589b12 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp +++ b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/index/pk/primary_key_index_source_meta.h" diff --git a/src/paimon/core/index/pk/primary_key_index_source_policy.h b/src/paimon/core/index/pk/primary_key_index_source_policy.h index 410461e8d..5251bf9a9 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_policy.h +++ b/src/paimon/core/index/pk/primary_key_index_source_policy.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp index d9fbf2026..1751eef7f 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/index/pksorted/pk_sorted_bucket_index_state.h" diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h index fa4ee8f5e..42250288d 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp index 25dbaa83b..8e4c00521 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/index/pksorted/pk_sorted_bucket_index_state.h" diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp index 4a428b82e..386f0a6f9 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/index/pksorted/pk_sorted_index_file.h" diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.h b/src/paimon/core/index/pksorted/pk_sorted_index_file.h index 7f350314f..1c5089e94 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_file.h +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp index c0052d8b1..b226dc11d 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/index/pksorted/pk_sorted_index_group.h" diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_group.h b/src/paimon/core/index/pksorted/pk_sorted_index_group.h index 01a2a67cd..e202f8956 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_group.h +++ b/src/paimon/core/index/pksorted/pk_sorted_index_group.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp index 4b5c48b74..c529d5c18 100644 --- a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/table/source/primary_key_index_batch_scan.h" diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.h b/src/paimon/core/table/source/primary_key_index_batch_scan.h index d330de703..bf5284246 100644 --- a/src/paimon/core/table/source/primary_key_index_batch_scan.h +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp index 4e77f5c7d..7755c0994 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/table/source/primary_key_sorted_index_result.h" diff --git a/src/paimon/core/table/source/primary_key_sorted_index_result.h b/src/paimon/core/table/source/primary_key_sorted_index_result.h index bd6ed262d..8de494ad8 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_result.h +++ b/src/paimon/core/table/source/primary_key_sorted_index_result.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp index 328c69110..23684007a 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/table/source/primary_key_sorted_index_scan.h" diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.h b/src/paimon/core/table/source/primary_key_sorted_index_scan.h index 64e3fb038..e4838eb1c 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.h +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.h @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #pragma once diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index be1552a2c..0e211d526 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -1,17 +1,20 @@ /* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ #include "paimon/core/table/source/primary_key_sorted_index_scan.h" From a507f439186fe00744d3965d3b4166cc214cca47 Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Tue, 11 Aug 2026 05:58:38 -0400 Subject: [PATCH 07/14] fix(pk-index): satisfy release and clang-tidy builds --- .../common/utils/java_modified_utf8.cpp | 12 +++---- .../pk/primary_key_index_source_meta.cpp | 10 +++--- .../primary_key_sorted_index_result.cpp | 32 +++++++++++-------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/paimon/common/utils/java_modified_utf8.cpp b/src/paimon/common/utils/java_modified_utf8.cpp index 554cd1637..fc476d70d 100644 --- a/src/paimon/common/utils/java_modified_utf8.cpp +++ b/src/paimon/common/utils/java_modified_utf8.cpp @@ -52,7 +52,7 @@ Result JavaModifiedUtf8::Encode(std::string_view utf8) { out.reserve(utf8.size()); size_t i = 0; while (i < utf8.size()) { - uint8_t byte0 = static_cast(utf8[i]); + auto byte0 = static_cast(utf8[i]); if (byte0 < 0x80) { if (byte0 == 0) { // Java encodes U+0000 as the overlong two-byte form. @@ -81,7 +81,7 @@ Result JavaModifiedUtf8::Encode(std::string_view utf8) { return MalformedInput("truncated sequence", i); } for (int32_t k = 1; k <= continuation_count; k++) { - uint8_t continuation = static_cast(utf8[i + k]); + auto continuation = static_cast(utf8[i + k]); if ((continuation & 0xC0) != 0x80) { return MalformedInput("invalid continuation byte", i + k); } @@ -116,7 +116,7 @@ Result JavaModifiedUtf8::Decode(std::string_view modified_utf8) { uint32_t pending_high_surrogate = 0; bool has_pending_high_surrogate = false; while (i < modified_utf8.size()) { - uint8_t byte0 = static_cast(modified_utf8[i]); + auto byte0 = static_cast(modified_utf8[i]); uint32_t unit = 0; if (byte0 < 0x80) { if (byte0 == 0) { @@ -129,7 +129,7 @@ Result JavaModifiedUtf8::Decode(std::string_view modified_utf8) { if (i + 1 >= modified_utf8.size()) { return MalformedInput("truncated two-byte sequence", i); } - uint8_t byte1 = static_cast(modified_utf8[i + 1]); + auto byte1 = static_cast(modified_utf8[i + 1]); if ((byte1 & 0xC0) != 0x80) { return MalformedInput("invalid continuation byte", i + 1); } @@ -139,8 +139,8 @@ Result JavaModifiedUtf8::Decode(std::string_view modified_utf8) { if (i + 2 >= modified_utf8.size()) { return MalformedInput("truncated three-byte sequence", i); } - uint8_t byte1 = static_cast(modified_utf8[i + 1]); - uint8_t byte2 = static_cast(modified_utf8[i + 2]); + auto byte1 = static_cast(modified_utf8[i + 1]); + auto byte2 = static_cast(modified_utf8[i + 2]); if ((byte1 & 0xC0) != 0x80 || (byte2 & 0xC0) != 0x80) { return MalformedInput("invalid continuation byte", i + 1); } diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp index f70b8f78a..4c09b3f65 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp @@ -37,7 +37,7 @@ constexpr size_t kMinBytesPerSourceFile = sizeof(uint16_t) + sizeof(int64_t); constexpr size_t kMaxInitialSourceFileCapacity = 1024; void AppendBigEndian32(int32_t value, std::string* out) { - uint32_t bits = static_cast(value); + auto bits = static_cast(value); out->push_back(static_cast((bits >> 24) & 0xFF)); out->push_back(static_cast((bits >> 16) & 0xFF)); out->push_back(static_cast((bits >> 8) & 0xFF)); @@ -45,7 +45,7 @@ void AppendBigEndian32(int32_t value, std::string* out) { } void AppendBigEndian64(int64_t value, std::string* out) { - uint64_t bits = static_cast(value); + auto bits = static_cast(value); for (int32_t shift = 56; shift >= 0; shift -= 8) { out->push_back(static_cast((bits >> shift) & 0xFF)); } @@ -77,8 +77,8 @@ class BigEndianCursor { Result ReadUint16() { PAIMON_RETURN_NOT_OK(CheckAvailable(sizeof(uint16_t))); - uint16_t bits = static_cast((static_cast(data_[position_]) << 8) | - static_cast(data_[position_ + 1])); + auto bits = static_cast((static_cast(data_[position_]) << 8) | + static_cast(data_[position_ + 1])); position_ += sizeof(uint16_t); return bits; } @@ -186,7 +186,7 @@ Result> PrimaryKeyIndexSourceMeta::Serialize(MemoryPool* return Status::Invalid(fmt::format( "Source file name is too long for writeUTF: {} bytes.", encoded_name.size())); } - uint16_t name_length = static_cast(encoded_name.size()); + auto name_length = static_cast(encoded_name.size()); buffer.push_back(static_cast((name_length >> 8) & 0xFF)); buffer.push_back(static_cast(name_length & 0xFF)); buffer.append(encoded_name); diff --git a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp index 7755c0994..2814b3325 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp @@ -21,7 +21,6 @@ #include #include -#include #include #include @@ -30,6 +29,11 @@ namespace paimon { namespace { +struct RangeConversion { + bool use_index; + std::vector ranges; +}; + Result> ToSingleFileSplit( const PrimaryKeySortedIndexScan::FilePlan& file) { const std::shared_ptr& source = file.SourceSplit(); @@ -46,11 +50,10 @@ Result> ToSingleFileSplit( return builder.Build(); } -/// Converts sorted file-local positions to merged ranges. Returns `std::nullopt` when a -/// position is invalid or the result is over-fragmented, in which case the file must fall -/// back to a normal scan. -Result>> ToRanges(const GlobalIndexResult& result, - int64_t row_count) { +/// Converts sorted file-local positions to merged ranges. Sets `use_index` to false when a +/// position is invalid or the result is over-fragmented, in which case the file must fall back +/// to a normal scan. +Result ToRanges(const GlobalIndexResult& result, int64_t row_count) { std::vector ranges; int64_t from = -1; int64_t to = -1; @@ -60,14 +63,14 @@ Result>> ToRanges(const GlobalIndexResult& resu int64_t position = iterator->Next(); if (position < 0 || position >= row_count || position >= std::numeric_limits::max()) { - return std::optional>(); + return RangeConversion{/*use_index=*/false, {}}; } if (from < 0) { from = position; } else if (position != to + 1) { if (ranges.size() >= static_cast(PrimaryKeySortedIndexResult::kMaxIndexedRangesPerFile)) { - return std::optional>(); + return RangeConversion{/*use_index=*/false, {}}; } ranges.emplace_back(from, to); from = position; @@ -76,10 +79,10 @@ Result>> ToRanges(const GlobalIndexResult& resu } if (ranges.size() >= static_cast(PrimaryKeySortedIndexResult::kMaxIndexedRangesPerFile)) { - return std::optional>(); + return RangeConversion{/*use_index=*/false, {}}; } ranges.emplace_back(from, to); - return std::optional>(std::move(ranges)); + return RangeConversion{/*use_index=*/true, std::move(ranges)}; } } // namespace @@ -123,17 +126,18 @@ Result>> PrimaryKeySortedIndexResult::ToSplit if (is_empty) { continue; } - PAIMON_ASSIGN_OR_RAISE(std::optional> ranges, + PAIMON_ASSIGN_OR_RAISE(RangeConversion range_conversion, ToRanges(*result, file.DataFile()->row_count)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr single_file_split, ToSingleFileSplit(file)); - if (ranges == std::nullopt) { + if (!range_conversion.use_index) { // The index returned an invalid or over-fragmented row position set; fall back // to a normal scan for this file. splits.push_back(std::move(single_file_split)); } else { - splits.push_back(std::make_shared( - std::move(single_file_split), std::move(ranges).value(), std::vector())); + splits.push_back(std::make_shared(std::move(single_file_split), + std::move(range_conversion.ranges), + std::vector())); } } return splits; From 1469b2e02aadc1ceb6c8f39067914522192efe9a Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Thu, 13 Aug 2026 08:15:34 -0400 Subject: [PATCH 08/14] fix(pk-index): address review feedback --- .../user_guide/primary_key_global_index.rst | 7 +- include/paimon/global_index/global_indexer.h | 23 +++ src/paimon/CMakeLists.txt | 2 - .../btree_global_index_integration_test.cpp | 32 ++- .../btree/btree_global_indexer.cpp | 11 +- .../global_index/btree/btree_global_indexer.h | 5 + .../common/utils/java_modified_utf8.cpp | 190 ------------------ src/paimon/common/utils/java_modified_utf8.h | 49 ----- .../common/utils/java_modified_utf8_test.cpp | 119 ----------- .../index/pk/primary_key_index_definition.h | 16 +- .../pk/primary_key_index_definitions.cpp | 74 ++++--- .../pk/primary_key_index_definitions_test.cpp | 6 +- .../pk/primary_key_index_source_meta.cpp | 171 ++++++---------- .../index/pk/primary_key_index_source_meta.h | 12 +- .../pk/primary_key_index_source_meta_test.cpp | 31 ++- .../pk_sorted_bucket_index_state_test.cpp | 7 +- .../index/pksorted/pk_sorted_index_file.cpp | 3 +- .../source/primary_key_index_batch_scan.cpp | 25 ++- .../source/primary_key_sorted_index_scan.cpp | 7 +- .../source/primary_key_sorted_index_scan.h | 5 +- .../primary_key_sorted_index_scan_test.cpp | 170 +++++++--------- 21 files changed, 340 insertions(+), 625 deletions(-) delete mode 100644 src/paimon/common/utils/java_modified_utf8.cpp delete mode 100644 src/paimon/common/utils/java_modified_utf8.h delete mode 100644 src/paimon/common/utils/java_modified_utf8_test.cpp diff --git a/docs/source/user_guide/primary_key_global_index.rst b/docs/source/user_guide/primary_key_global_index.rst index b820dcdce..3b9aaa56e 100644 --- a/docs/source/user_guide/primary_key_global_index.rst +++ b/docs/source/user_guide/primary_key_global_index.rst @@ -63,8 +63,11 @@ Current scope - The BTree payload reader is wired up. ``pk-bitmap`` (and vector / full-text) definitions are recognized for validation, but their evaluation conservatively falls back to a normal scan until their dedicated payload readers are supported. -- The read path targets the Java release-2.0.0 wire contract (source metadata v1, - ``GlobalIndexMeta`` with ``_SOURCE_META``, commit message v12). +- The read path targets the Java release-2.0.0 layout and scan semantics (source metadata + v1, ``GlobalIndexMeta`` with ``_SOURCE_META``, commit message v12). Source-file names + currently use the existing C++ length-prefixed UTF-8 streams; ASCII and non-null BMP + names are compatible with Java ``writeUTF``, while complete modified UTF-8 support for + supplementary code points will be handled by a shared stream-level change. - ``PkSortedIndexFile::Build`` can build one payload for an ordered source group from value-sorted input, which supports tooling and tests; automatic build and maintenance during compaction is not included yet. diff --git a/include/paimon/global_index/global_indexer.h b/include/paimon/global_index/global_indexer.h index 4da6293ff..690ec4bd1 100644 --- a/include/paimon/global_index/global_indexer.h +++ b/include/paimon/global_index/global_indexer.h @@ -35,6 +35,8 @@ struct ArrowSchema; namespace paimon { +class Executor; + /// Interface for creating global index readers and writers. class PAIMON_EXPORT GlobalIndexer { public: @@ -70,6 +72,27 @@ class PAIMON_EXPORT GlobalIndexer { ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, const std::vector& files, const std::shared_ptr& pool) const = 0; + + /// Creates a reader using an executor supplied by the scan layer. + /// + /// Index implementations which do not perform asynchronous work may ignore the executor and + /// use the compatibility overload above. + /// + /// @param arrow_schema Schema of the indexed data; used to interpret predicate literals. + /// @param file_reader I/O handler for reading index artifacts from storage. + /// @param files List of index file metadata entries produced during writing. + /// @param pool Memory pool for temporary allocations; if nullptr, uses default. + /// @param executor Executor shared by readers created for the same scan; nullptr means + /// that the reader should evaluate sequentially. + /// @return A `Result` containing a shared pointer to the created `GlobalIndexReader`, + /// or an error if the index cannot be loaded or is incompatible, etc. + virtual Result> CreateReader( + ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, const std::shared_ptr& pool, + const std::shared_ptr& executor) const { + static_cast(executor); + return CreateReader(arrow_schema, file_reader, files, pool); + } }; } // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index bf2ce2c4d..495bd4e52 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -175,7 +175,6 @@ set(PAIMON_COMMON_SRCS common/data/shredding/shredding_file_reader.cpp common/utils/delta_varint_compressor.cpp common/utils/fields_comparator.cpp - common/utils/java_modified_utf8.cpp common/utils/path_util.cpp common/utils/range.cpp common/utils/read_ahead_cache.cpp @@ -492,7 +491,6 @@ if(PAIMON_BUILD_TESTS) SOURCES common/memory/memory_pool_test.cpp common/memory/bytes_test.cpp - common/utils/java_modified_utf8_test.cpp common/memory/memory_segment_test.cpp common/memory/memory_segment_utils_test.cpp common/memory/memory_slice_test.cpp diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index e5aeaac20..9bfb97e0c 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -16,6 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +#include +#include +#include + #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" @@ -29,6 +33,7 @@ #include "paimon/common/utils/scope_guard.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" +#include "paimon/executor.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/io/global_index_file_reader.h" @@ -84,6 +89,27 @@ class FakeGlobalIndexFileReader : public GlobalIndexFileReader { std::string base_path_; }; +class CountingInlineExecutor : public Executor { + public: + void Add(std::function func) override { + submission_count_.fetch_add(1); + func(); + } + + void ShutdownNow() override {} + + uint32_t GetThreadNum() const override { + return 1; + } + + uint32_t SubmissionCount() const { + return submission_count_.load(); + } + + private: + std::atomic submission_count_{0}; +}; + class BTreeGlobalIndexIntegrationTest : public ::testing::Test, public ::testing::WithParamInterface { protected: @@ -1972,9 +1998,10 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadMultiFilesWithMetaSelector) // Create reader over all 3 files (internally uses LazyFilteredBTreeReader + // BTreeFileMetaSelector) auto file_reader = std::make_shared(fs_, base_path_); + auto executor = std::make_shared(); auto c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, all_metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, all_metas, + pool_, executor)); // --- VisitEqual: key=12 -> only file1 is selected by meta selector -> row 5 { @@ -2023,6 +2050,7 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadMultiFilesWithMetaSelector) Literal literal_5(5); ASSERT_OK_AND_ASSIGN(auto result, reader->VisitGreaterOrEqual(literal_5)); CheckResult(result, {3, 4, 5, 6, 7, 9}); + ASSERT_EQ(executor->SubmissionCount(), 3); } // --- VisitLessOrEqual: key <= 2 -> only file0 selected -> rows 0,1 diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index f3e97b6ed..062b27e61 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -38,7 +38,6 @@ #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/preconditions.h" #include "paimon/core/options/compress_options.h" -#include "paimon/executor.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/memory/bytes.h" #include "paimon/utils/roaring_bitmap64.h" @@ -103,6 +102,13 @@ Result> BTreeGlobalIndexer::CreateWriter( Result> BTreeGlobalIndexer::CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, const std::vector& files, const std::shared_ptr& pool) const { + return CreateReader(arrow_schema, file_reader, files, pool, /*executor=*/nullptr); +} + +Result> BTreeGlobalIndexer::CreateReader( + ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, const std::shared_ptr& pool, + const std::shared_ptr& executor) const { // Get field type from arrow schema PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, arrow::ImportSchema(arrow_schema)); @@ -123,9 +129,6 @@ Result> BTreeGlobalIndexer::CreateReader( } read_buffer_size = static_cast(tmp_buffer_size); } - // Readers are created per payload group and may coexist for many buckets. Share the - // process-wide executor instead of creating a dedicated thread pool for every group. - std::shared_ptr executor = GetGlobalDefaultExecutor(); return std::make_shared(read_buffer_size, files, key_type, file_reader, cache_manager_, pool, executor); } diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.h b/src/paimon/common/global_index/btree/btree_global_indexer.h index 5568adba3..f93099487 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.h +++ b/src/paimon/common/global_index/btree/btree_global_indexer.h @@ -68,6 +68,11 @@ class BTreeGlobalIndexer : public GlobalIndexer { const std::vector& files, const std::shared_ptr& pool) const override; + Result> CreateReader( + ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, const std::shared_ptr& pool, + const std::shared_ptr& executor) const override; + private: BTreeGlobalIndexer(const std::shared_ptr& cache_manager, const std::map& options) diff --git a/src/paimon/common/utils/java_modified_utf8.cpp b/src/paimon/common/utils/java_modified_utf8.cpp deleted file mode 100644 index fc476d70d..000000000 --- a/src/paimon/common/utils/java_modified_utf8.cpp +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "paimon/common/utils/java_modified_utf8.h" - -#include - -#include "fmt/format.h" - -namespace paimon { -namespace { -constexpr uint32_t kSupplementaryStart = 0x10000; -constexpr uint32_t kMaxCodePoint = 0x10FFFF; -constexpr uint32_t kHighSurrogateStart = 0xD800; -constexpr uint32_t kLowSurrogateStart = 0xDC00; -constexpr uint32_t kSurrogateEnd = 0xDFFF; - -void AppendTwoBytes(uint32_t code_point, std::string* out) { - out->push_back(static_cast(0xC0 | ((code_point >> 6) & 0x1F))); - out->push_back(static_cast(0x80 | (code_point & 0x3F))); -} - -void AppendThreeBytes(uint32_t code_point, std::string* out) { - out->push_back(static_cast(0xE0 | ((code_point >> 12) & 0x0F))); - out->push_back(static_cast(0x80 | ((code_point >> 6) & 0x3F))); - out->push_back(static_cast(0x80 | (code_point & 0x3F))); -} - -Status MalformedInput(const std::string& what, size_t position) { - return Status::Invalid(fmt::format("Malformed UTF-8 input: {} around byte {}", what, position)); -} -} // namespace - -Result JavaModifiedUtf8::Encode(std::string_view utf8) { - std::string out; - out.reserve(utf8.size()); - size_t i = 0; - while (i < utf8.size()) { - auto byte0 = static_cast(utf8[i]); - if (byte0 < 0x80) { - if (byte0 == 0) { - // Java encodes U+0000 as the overlong two-byte form. - AppendTwoBytes(0, &out); - } else { - out.push_back(static_cast(byte0)); - } - i += 1; - continue; - } - int32_t continuation_count = 0; - uint32_t code_point = 0; - if ((byte0 & 0xE0) == 0xC0) { - continuation_count = 1; - code_point = byte0 & 0x1F; - } else if ((byte0 & 0xF0) == 0xE0) { - continuation_count = 2; - code_point = byte0 & 0x0F; - } else if ((byte0 & 0xF8) == 0xF0) { - continuation_count = 3; - code_point = byte0 & 0x07; - } else { - return MalformedInput("invalid leading byte", i); - } - if (i + continuation_count >= utf8.size()) { - return MalformedInput("truncated sequence", i); - } - for (int32_t k = 1; k <= continuation_count; k++) { - auto continuation = static_cast(utf8[i + k]); - if ((continuation & 0xC0) != 0x80) { - return MalformedInput("invalid continuation byte", i + k); - } - code_point = (code_point << 6) | (continuation & 0x3F); - } - // Reject overlong forms and code points outside Unicode. - static constexpr uint32_t kMinByLength[4] = {0, 0x80, 0x800, kSupplementaryStart}; - if (code_point < kMinByLength[continuation_count] || code_point > kMaxCodePoint || - (code_point >= kHighSurrogateStart && code_point <= kSurrogateEnd)) { - return MalformedInput("invalid code point", i); - } - if (code_point < 0x800) { - AppendTwoBytes(code_point, &out); - } else if (code_point < kSupplementaryStart) { - AppendThreeBytes(code_point, &out); - } else { - // Java writes supplementary code points as a CESU-8 surrogate pair. - uint32_t offset = code_point - kSupplementaryStart; - AppendThreeBytes(kHighSurrogateStart + (offset >> 10), &out); - AppendThreeBytes(kLowSurrogateStart + (offset & 0x3FF), &out); - } - i += 1 + continuation_count; - } - return out; -} - -Result JavaModifiedUtf8::Decode(std::string_view modified_utf8) { - std::string out; - out.reserve(modified_utf8.size()); - size_t i = 0; - // Decoded UTF-16 code units, kept across iterations to pair surrogates. - uint32_t pending_high_surrogate = 0; - bool has_pending_high_surrogate = false; - while (i < modified_utf8.size()) { - auto byte0 = static_cast(modified_utf8[i]); - uint32_t unit = 0; - if (byte0 < 0x80) { - if (byte0 == 0) { - // Java's writeUTF never emits a raw zero byte. - return MalformedInput("unexpected raw zero byte", i); - } - unit = byte0; - i += 1; - } else if ((byte0 & 0xE0) == 0xC0) { - if (i + 1 >= modified_utf8.size()) { - return MalformedInput("truncated two-byte sequence", i); - } - auto byte1 = static_cast(modified_utf8[i + 1]); - if ((byte1 & 0xC0) != 0x80) { - return MalformedInput("invalid continuation byte", i + 1); - } - unit = ((byte0 & 0x1F) << 6) | (byte1 & 0x3F); - i += 2; - } else if ((byte0 & 0xF0) == 0xE0) { - if (i + 2 >= modified_utf8.size()) { - return MalformedInput("truncated three-byte sequence", i); - } - auto byte1 = static_cast(modified_utf8[i + 1]); - auto byte2 = static_cast(modified_utf8[i + 2]); - if ((byte1 & 0xC0) != 0x80 || (byte2 & 0xC0) != 0x80) { - return MalformedInput("invalid continuation byte", i + 1); - } - unit = ((byte0 & 0x0F) << 12) | ((byte1 & 0x3F) << 6) | (byte2 & 0x3F); - i += 3; - } else { - // Java's readUTF rejects four-byte sequences and stray continuation bytes. - return MalformedInput("invalid leading byte", i); - } - - if (has_pending_high_surrogate) { - if (unit >= kLowSurrogateStart && unit <= kSurrogateEnd) { - uint32_t code_point = kSupplementaryStart + - ((pending_high_surrogate - kHighSurrogateStart) << 10) + - (unit - kLowSurrogateStart); - out.push_back(static_cast(0xF0 | ((code_point >> 18) & 0x07))); - out.push_back(static_cast(0x80 | ((code_point >> 12) & 0x3F))); - out.push_back(static_cast(0x80 | ((code_point >> 6) & 0x3F))); - out.push_back(static_cast(0x80 | (code_point & 0x3F))); - has_pending_high_surrogate = false; - continue; - } - return MalformedInput("unpaired high surrogate", i); - } - if (unit >= kHighSurrogateStart && unit < kLowSurrogateStart) { - pending_high_surrogate = unit; - has_pending_high_surrogate = true; - continue; - } - if (unit >= kLowSurrogateStart && unit <= kSurrogateEnd) { - return MalformedInput("unpaired low surrogate", i); - } - if (unit < 0x80) { - out.push_back(static_cast(unit)); - } else if (unit < 0x800) { - AppendTwoBytes(unit, &out); - } else { - AppendThreeBytes(unit, &out); - } - } - if (has_pending_high_surrogate) { - return MalformedInput("unpaired high surrogate at end", modified_utf8.size()); - } - return out; -} - -} // namespace paimon diff --git a/src/paimon/common/utils/java_modified_utf8.h b/src/paimon/common/utils/java_modified_utf8.h deleted file mode 100644 index 84dbfe537..000000000 --- a/src/paimon/common/utils/java_modified_utf8.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include -#include - -#include "paimon/result.h" - -namespace paimon { -/// Converts between standard UTF-8 and the "modified UTF-8" used by Java's -/// `DataOutputStream#writeUTF` / `DataInputStream#readUTF`: -/// - U+0000 is encoded as the two-byte sequence 0xC0 0x80 instead of a single zero byte; -/// - supplementary code points (U+10000 and above) are encoded as a UTF-16 surrogate pair, -/// each surrogate written as an independent three-byte sequence (CESU-8), instead of the -/// four-byte standard UTF-8 form. -class JavaModifiedUtf8 { - public: - JavaModifiedUtf8() = delete; - ~JavaModifiedUtf8() = delete; - - /// Encodes a standard UTF-8 string into Java modified UTF-8 bytes. - /// @return An error status if `utf8` is not well-formed UTF-8. - static Result Encode(std::string_view utf8); - - /// Decodes Java modified UTF-8 bytes into a standard UTF-8 string, mirroring the - /// validation of Java's `DataInputStream#readUTF`. - /// @return An error status on any malformed byte sequence. - static Result Decode(std::string_view modified_utf8); -}; - -} // namespace paimon diff --git a/src/paimon/common/utils/java_modified_utf8_test.cpp b/src/paimon/common/utils/java_modified_utf8_test.cpp deleted file mode 100644 index 6eaecc5b5..000000000 --- a/src/paimon/common/utils/java_modified_utf8_test.cpp +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "paimon/common/utils/java_modified_utf8.h" - -#include - -#include "gtest/gtest.h" -#include "paimon/testing/utils/testharness.h" - -namespace paimon::test { - -TEST(JavaModifiedUtf8Test, AsciiEncodeIsIdentity) { - std::string ascii = "data-8b2f1a-0.parquet"; - ASSERT_OK_AND_ASSIGN(std::string encoded, JavaModifiedUtf8::Encode(ascii)); - ASSERT_EQ(ascii, encoded); - ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode(encoded)); - ASSERT_EQ(ascii, decoded); -} - -TEST(JavaModifiedUtf8Test, BmpTextRoundTrip) { - // Chinese characters are three-byte sequences, identical in both encodings. - std::string utf8 = "订单表-文件.parquet"; - ASSERT_OK_AND_ASSIGN(std::string encoded, JavaModifiedUtf8::Encode(utf8)); - ASSERT_EQ(utf8, encoded); - ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode(encoded)); - ASSERT_EQ(utf8, decoded); -} - -TEST(JavaModifiedUtf8Test, NulByteUsesOverlongTwoByteForm) { - std::string nul(1, '\0'); - ASSERT_OK_AND_ASSIGN(std::string encoded, JavaModifiedUtf8::Encode(nul)); - ASSERT_EQ("\xC0\x80", encoded); - ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode("\xC0\x80")); - ASSERT_EQ(nul, decoded); - - // U+0000 embedded in surrounding ASCII leaves its neighbors untouched. - std::string embedded("ab\0cd", 5); - ASSERT_OK_AND_ASSIGN(std::string embedded_encoded, JavaModifiedUtf8::Encode(embedded)); - ASSERT_EQ(std::string("ab\xC0\x80" - "cd", - 6), - embedded_encoded); - ASSERT_OK_AND_ASSIGN(std::string embedded_decoded, JavaModifiedUtf8::Decode(embedded_encoded)); - ASSERT_EQ(embedded, embedded_decoded); -} - -TEST(JavaModifiedUtf8Test, SupplementaryCharUsesSurrogatePair) { - // U+1F600 in standard four-byte UTF-8. - std::string standard = "\xF0\x9F\x98\x80"; - ASSERT_OK_AND_ASSIGN(std::string encoded, JavaModifiedUtf8::Encode(standard)); - // CESU-8: surrogate pair U+D83D U+DE00, each written as a three-byte sequence. - ASSERT_EQ("\xED\xA0\xBD\xED\xB8\x80", encoded); - ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode(encoded)); - ASSERT_EQ(standard, decoded); -} - -TEST(JavaModifiedUtf8Test, DecodeRejectsMalformedInput) { - // Java's writeUTF never emits a raw zero byte. - ASSERT_NOK(JavaModifiedUtf8::Decode(std::string(1, '\0'))); - ASSERT_NOK(JavaModifiedUtf8::Decode(std::string("a\0b", 3))); - // Truncated two-byte and three-byte sequences. - ASSERT_NOK(JavaModifiedUtf8::Decode("\xC3")); - ASSERT_NOK(JavaModifiedUtf8::Decode("\xE8\xB8")); - // Continuation bytes must match 10xxxxxx. - ASSERT_NOK(JavaModifiedUtf8::Decode("\xC3\x28")); - ASSERT_NOK(JavaModifiedUtf8::Decode("\xE8\x28\xB8")); - // readUTF rejects four-byte leading bytes; supplementary chars must arrive as CESU-8. - ASSERT_NOK(JavaModifiedUtf8::Decode("\xF0\x9F\x98\x80")); - // Unpaired high surrogate at end of input. - ASSERT_NOK(JavaModifiedUtf8::Decode("\xED\xA0\xBD")); - // High surrogate followed by a non-surrogate unit. - ASSERT_NOK( - JavaModifiedUtf8::Decode("\xED\xA0\xBD" - "z")); - // Low surrogate without a preceding high surrogate. - ASSERT_NOK(JavaModifiedUtf8::Decode("\xED\xB8\x80")); -} - -TEST(JavaModifiedUtf8Test, EncodeRejectsInvalidUtf8) { - // Stray continuation byte. - ASSERT_NOK(JavaModifiedUtf8::Encode("\x80")); - ASSERT_NOK(JavaModifiedUtf8::Encode("a\x80")); - // Truncated multi-byte sequences. - ASSERT_NOK(JavaModifiedUtf8::Encode("\xC3")); - ASSERT_NOK(JavaModifiedUtf8::Encode("\xE8\xB8")); - ASSERT_NOK(JavaModifiedUtf8::Encode("\xF0\x9F\x98")); - // Overlong two-byte encoding of U+002F. - ASSERT_NOK(JavaModifiedUtf8::Encode("\xC0\xAF")); - // Surrogate code point U+D800 encoded directly as a three-byte sequence. - ASSERT_NOK(JavaModifiedUtf8::Encode("\xED\xA0\x80")); - // Code point above U+10FFFF. - ASSERT_NOK(JavaModifiedUtf8::Encode("\xF4\x90\x80\x80")); -} - -TEST(JavaModifiedUtf8Test, DecodeAcceptsJavaLenientOverlongTwoByteForm) { - // Java's readUTF only pattern-matches the bit layout of two-byte sequences, so the - // overlong encoding 0xC1 0xBF of U+007F is accepted; Decode mirrors that leniency. - ASSERT_OK_AND_ASSIGN(std::string decoded, JavaModifiedUtf8::Decode("\xC1\xBF")); - ASSERT_EQ("\x7F", decoded); -} - -} // namespace paimon::test diff --git a/src/paimon/core/index/pk/primary_key_index_definition.h b/src/paimon/core/index/pk/primary_key_index_definition.h index 098903e1c..1a06628e9 100644 --- a/src/paimon/core/index/pk/primary_key_index_definition.h +++ b/src/paimon/core/index/pk/primary_key_index_definition.h @@ -37,12 +37,12 @@ class PrimaryKeyIndexDefinition { }; PrimaryKeyIndexDefinition(std::string column, int32_t field_id, std::string index_type, - std::map options, Family family) + Family family, std::map options) : column_(std::move(column)), field_id_(field_id), index_type_(std::move(index_type)), - options_(std::move(options)), - family_(family) {} + family_(family), + options_(std::move(options)) {} const std::string& Column() const { return column_; @@ -56,20 +56,20 @@ class PrimaryKeyIndexDefinition { return index_type_; } - const std::map& Options() const { - return options_; - } - Family GetFamily() const { return family_; } + const std::map& Options() const { + return options_; + } + private: std::string column_; int32_t field_id_; std::string index_type_; - std::map options_; Family family_; + std::map options_; }; } // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.cpp b/src/paimon/core/index/pk/primary_key_index_definitions.cpp index 57fd45ae0..8f87184ba 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions.cpp +++ b/src/paimon/core/index/pk/primary_key_index_definitions.cpp @@ -19,6 +19,7 @@ #include "paimon/core/index/pk/primary_key_index_definitions.h" +#include #include #include @@ -32,6 +33,8 @@ namespace paimon { namespace { +using IndexOptions = std::map; + constexpr char kBTreeIndexType[] = "btree"; constexpr char kBitmapIndexType[] = "bitmap"; constexpr char kFullTextIndexType[] = "full-text"; @@ -55,26 +58,37 @@ std::vector IndexColumns(const std::map& return columns; } -Status ValidateNoDuplicates(const std::vector& columns, const char* option_key) { - std::set unique_columns; +Status AddUniqueColumns(const std::vector& columns, + const std::function& on_duplicate, + std::set* unique_columns) { for (const std::string& column : columns) { - if (!unique_columns.insert(column).second) { - return Status::Invalid( - fmt::format("{} contains duplicate column '{}'.", option_key, column)); + if (!unique_columns->insert(column).second) { + return on_duplicate(column); } } return Status::OK(); } -Status ValidateUniqueColumns(std::set* indexed_columns, - const std::vector& columns) { - for (const std::string& column : columns) { - if (!indexed_columns->insert(column).second) { +Status ValidateNoDuplicates(const std::vector& columns, const char* option_key) { + std::set unique_columns; + return AddUniqueColumns( + columns, + [option_key](const std::string& column) { + return Status::Invalid( + fmt::format("{} contains duplicate column '{}'.", option_key, column)); + }, + &unique_columns); +} + +Status ValidateUniqueColumns(const std::vector& columns, + std::set* indexed_columns) { + return AddUniqueColumns( + columns, + [](const std::string& column) { return Status::Invalid( fmt::format("Column '{}' can own at most one primary-key index.", column)); - } - } - return Status::OK(); + }, + indexed_columns); } /// Resolves the effective option map of one sorted-index definition: table options first, @@ -154,28 +168,28 @@ Result PrimaryKeyIndexDefinitions::Create(const Tabl PAIMON_RETURN_NOT_OK( ValidateNoDuplicates(full_text_columns, Options::PK_FULL_TEXT_INDEX_COLUMNS)); std::set indexed_columns; - PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(&indexed_columns, vector_columns)); - PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(&indexed_columns, btree_columns)); - PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(&indexed_columns, bitmap_columns)); - PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(&indexed_columns, full_text_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(vector_columns, &indexed_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(btree_columns, &indexed_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(bitmap_columns, &indexed_columns)); + PAIMON_RETURN_NOT_OK(ValidateUniqueColumns(full_text_columns, &indexed_columns)); std::vector definitions; for (const DataField& field : schema.Fields()) { const std::string& column = field.Name(); if (ObjectUtils::Contains(btree_columns, column)) { - Result> definition_options = - SortedIndexOptions(options, column, kBTreeOptionFamily, kBTreeAlgorithmPrefix); - PAIMON_RETURN_NOT_OK(definition_options.status()); + PAIMON_ASSIGN_OR_RAISE( + IndexOptions definition_options, + SortedIndexOptions(options, column, kBTreeOptionFamily, kBTreeAlgorithmPrefix)); definitions.emplace_back(column, field.Id(), kBTreeIndexType, - std::move(definition_options).value(), - PrimaryKeyIndexDefinition::Family::BTREE); + PrimaryKeyIndexDefinition::Family::BTREE, + std::move(definition_options)); } else if (ObjectUtils::Contains(bitmap_columns, column)) { - Result> definition_options = - SortedIndexOptions(options, column, kBitmapOptionFamily, kBitmapAlgorithmPrefix); - PAIMON_RETURN_NOT_OK(definition_options.status()); + PAIMON_ASSIGN_OR_RAISE( + IndexOptions definition_options, + SortedIndexOptions(options, column, kBitmapOptionFamily, kBitmapAlgorithmPrefix)); definitions.emplace_back(column, field.Id(), kBitmapIndexType, - std::move(definition_options).value(), - PrimaryKeyIndexDefinition::Family::BITMAP); + PrimaryKeyIndexDefinition::Family::BITMAP, + std::move(definition_options)); } else if (ObjectUtils::Contains(vector_columns, column)) { std::string index_type; auto type_iter = @@ -184,12 +198,12 @@ Result PrimaryKeyIndexDefinitions::Create(const Tabl index_type = type_iter->second; } definitions.emplace_back(column, field.Id(), index_type, - std::map(), - PrimaryKeyIndexDefinition::Family::VECTOR); + PrimaryKeyIndexDefinition::Family::VECTOR, + std::map()); } else if (ObjectUtils::Contains(full_text_columns, column)) { definitions.emplace_back(column, field.Id(), kFullTextIndexType, - std::map(), - PrimaryKeyIndexDefinition::Family::FULL_TEXT); + PrimaryKeyIndexDefinition::Family::FULL_TEXT, + std::map()); } } return PrimaryKeyIndexDefinitions(std::move(definitions)); diff --git a/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp b/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp index 8649e26d8..616b0dc98 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp +++ b/src/paimon/core/index/pk/primary_key_index_definitions_test.cpp @@ -178,14 +178,16 @@ TEST(PrimaryKeyIndexDefinitionsTest, RejectsMalformedJsonOptions) { TEST(PrimaryKeyIndexDefinitionsTest, RejectsDuplicateColumnWithinFamily) { ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price,price"}})); - ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_NOK_WITH_MSG(PrimaryKeyIndexDefinitions::Create(*schema), + "pk-btree.index.columns contains duplicate column 'price'."); } TEST(PrimaryKeyIndexDefinitionsTest, RejectsColumnSharedAcrossFamilies) { ASSERT_OK_AND_ASSIGN(std::unique_ptr schema, MakeSchema({{Options::PK_BTREE_INDEX_COLUMNS, "price"}, {Options::PK_BITMAP_INDEX_COLUMNS, "price"}})); - ASSERT_NOK(PrimaryKeyIndexDefinitions::Create(*schema)); + ASSERT_NOK_WITH_MSG(PrimaryKeyIndexDefinitions::Create(*schema), + "Column 'price' can own at most one primary-key index."); } TEST(PrimaryKeyIndexDefinitionsTest, ResolvesNonScalarFamiliesAndExcludesThemFromScalar) { diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp index 4c09b3f65..c31c34689 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta.cpp +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.cpp @@ -20,95 +20,25 @@ #include "paimon/core/index/pk/primary_key_index_source_meta.h" #include +#include #include +#include #include -#include #include #include "fmt/format.h" -#include "paimon/common/utils/java_modified_utf8.h" +#include "paimon/common/io/memory_segment_output_stream.h" +#include "paimon/common/memory/memory_segment_utils.h" #include "paimon/core/index/index_file_meta.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/io/data_input_stream.h" namespace paimon { namespace { -// Each serialized entry needs at least the two-byte writeUTF length and one int64 row count, +// Each serialized entry needs at least one uint16 string length and one int64 row count, // mirroring the defensive source file count cap of the Java deserializer. -constexpr size_t kMinBytesPerSourceFile = sizeof(uint16_t) + sizeof(int64_t); +constexpr int64_t kMinBytesPerSourceFile = sizeof(uint16_t) + sizeof(int64_t); constexpr size_t kMaxInitialSourceFileCapacity = 1024; - -void AppendBigEndian32(int32_t value, std::string* out) { - auto bits = static_cast(value); - out->push_back(static_cast((bits >> 24) & 0xFF)); - out->push_back(static_cast((bits >> 16) & 0xFF)); - out->push_back(static_cast((bits >> 8) & 0xFF)); - out->push_back(static_cast(bits & 0xFF)); -} - -void AppendBigEndian64(int64_t value, std::string* out) { - auto bits = static_cast(value); - for (int32_t shift = 56; shift >= 0; shift -= 8) { - out->push_back(static_cast((bits >> shift) & 0xFF)); - } -} - -class BigEndianCursor { - public: - BigEndianCursor(const char* data, size_t length) : data_(data), length_(length) {} - - Result ReadInt32() { - PAIMON_RETURN_NOT_OK(CheckAvailable(sizeof(int32_t))); - uint32_t bits = 0; - for (size_t k = 0; k < sizeof(int32_t); k++) { - bits = (bits << 8) | static_cast(data_[position_ + k]); - } - position_ += sizeof(int32_t); - return static_cast(bits); - } - - Result ReadInt64() { - PAIMON_RETURN_NOT_OK(CheckAvailable(sizeof(int64_t))); - uint64_t bits = 0; - for (size_t k = 0; k < sizeof(int64_t); k++) { - bits = (bits << 8) | static_cast(data_[position_ + k]); - } - position_ += sizeof(int64_t); - return static_cast(bits); - } - - Result ReadUint16() { - PAIMON_RETURN_NOT_OK(CheckAvailable(sizeof(uint16_t))); - auto bits = static_cast((static_cast(data_[position_]) << 8) | - static_cast(data_[position_ + 1])); - position_ += sizeof(uint16_t); - return bits; - } - - Result ReadBytes(size_t length) { - PAIMON_RETURN_NOT_OK(CheckAvailable(length)); - std::string_view view(data_ + position_, length); - position_ += length; - return view; - } - - size_t Available() const { - return length_ - position_; - } - - private: - Status CheckAvailable(size_t needed) const { - if (length_ - position_ < needed) { - return Status::Invalid(fmt::format( - "Failed to deserialize index source metadata: need {} bytes at offset {} but " - "only {} remain.", - needed, position_, length_ - position_)); - } - return Status::OK(); - } - - const char* data_; - size_t length_; - size_t position_ = 0; -}; } // namespace Result PrimaryKeyIndexSourceMeta::Create( @@ -141,18 +71,30 @@ Result PrimaryKeyIndexSourceMeta::FromIndexFile( Result PrimaryKeyIndexSourceMeta::Deserialize(const char* data, size_t length) { - BigEndianCursor cursor(data, length); - PAIMON_ASSIGN_OR_RAISE(int32_t version, cursor.ReadInt32()); + if (data == nullptr) { + return Status::Invalid("Cannot deserialize index source metadata from a null buffer."); + } + if (length > static_cast(std::numeric_limits::max())) { + return Status::Invalid( + fmt::format("Index source metadata length {} exceeds the supported maximum {}.", length, + std::numeric_limits::max())); + } + + auto input_stream = std::make_shared(data, static_cast(length)); + DataInputStream input(input_stream); + PAIMON_ASSIGN_OR_RAISE(int32_t version, input.ReadValue()); if (version != VERSION) { return Status::Invalid(fmt::format("Unsupported index source version: {}.", version)); } - PAIMON_ASSIGN_OR_RAISE(int32_t data_level, cursor.ReadInt32()); - PAIMON_ASSIGN_OR_RAISE(int32_t source_file_count, cursor.ReadInt32()); + PAIMON_ASSIGN_OR_RAISE(int32_t data_level, input.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int32_t source_file_count, input.ReadValue()); if (source_file_count <= 0) { return Status::Invalid("An index must reference source files."); } - size_t maximum_source_file_count = cursor.Available() / kMinBytesPerSourceFile; - if (static_cast(source_file_count) > maximum_source_file_count) { + PAIMON_ASSIGN_OR_RAISE(int64_t position, input.GetPos()); + PAIMON_ASSIGN_OR_RAISE(int64_t stream_length, input.Length()); + int64_t maximum_source_file_count = (stream_length - position) / kMinBytesPerSourceFile; + if (static_cast(source_file_count) > maximum_source_file_count) { return Status::Invalid(fmt::format( "Failed to deserialize index source metadata: source file count {} exceeds the " "maximum {} allowed by the remaining bytes.", @@ -162,37 +104,58 @@ Result PrimaryKeyIndexSourceMeta::Deserialize(const c source_files.reserve( std::min(static_cast(source_file_count), kMaxInitialSourceFileCapacity)); for (int32_t i = 0; i < source_file_count; i++) { - PAIMON_ASSIGN_OR_RAISE(uint16_t name_length, cursor.ReadUint16()); - PAIMON_ASSIGN_OR_RAISE(std::string_view name_bytes, cursor.ReadBytes(name_length)); - PAIMON_ASSIGN_OR_RAISE(std::string file_name, JavaModifiedUtf8::Decode(name_bytes)); - PAIMON_ASSIGN_OR_RAISE(int64_t row_count, cursor.ReadInt64()); + PAIMON_ASSIGN_OR_RAISE(std::string file_name, input.ReadString()); + PAIMON_ASSIGN_OR_RAISE(int64_t row_count, input.ReadValue()); source_files.emplace_back(std::move(file_name), row_count); } - if (cursor.Available() != 0) { + PAIMON_ASSIGN_OR_RAISE(position, input.GetPos()); + if (position != stream_length) { return Status::Invalid("Unexpected trailing bytes in index source metadata."); } return Create(data_level, std::move(source_files)); } -Result> PrimaryKeyIndexSourceMeta::Serialize(MemoryPool* pool) const { - std::string buffer; - AppendBigEndian32(VERSION, &buffer); - AppendBigEndian32(data_level_, &buffer); - AppendBigEndian32(static_cast(source_files_.size()), &buffer); +Result> PrimaryKeyIndexSourceMeta::Serialize( + const std::shared_ptr& pool) const { + if (pool == nullptr) { + return Status::Invalid("Cannot serialize index source metadata with a null memory pool."); + } + if (source_files_.size() > static_cast(std::numeric_limits::max())) { + return Status::Invalid( + fmt::format("Index source file count {} exceeds the supported maximum {}.", + source_files_.size(), std::numeric_limits::max())); + } + + int64_t serialized_size = 3 * static_cast(sizeof(int32_t)); for (const PrimaryKeyIndexSourceFile& source_file : source_files_) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_name, - JavaModifiedUtf8::Encode(source_file.file_name)); - if (encoded_name.size() > std::numeric_limits::max()) { + if (source_file.file_name.size() > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("Source file name is too long for a 16-bit length: {} bytes.", + source_file.file_name.size())); + } + int64_t entry_size = + kMinBytesPerSourceFile + static_cast(source_file.file_name.size()); + if (serialized_size > std::numeric_limits::max() - entry_size) { return Status::Invalid(fmt::format( - "Source file name is too long for writeUTF: {} bytes.", encoded_name.size())); + "Serialized index source metadata exceeds the supported maximum {} bytes.", + std::numeric_limits::max())); } - auto name_length = static_cast(encoded_name.size()); - buffer.push_back(static_cast((name_length >> 8) & 0xFF)); - buffer.push_back(static_cast(name_length & 0xFF)); - buffer.append(encoded_name); - AppendBigEndian64(source_file.row_count, &buffer); + serialized_size += entry_size; } - return std::make_shared(buffer, pool); + + MemorySegmentOutputStream output(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); + output.WriteValue(VERSION); + output.WriteValue(data_level_); + output.WriteValue(static_cast(source_files_.size())); + for (const PrimaryKeyIndexSourceFile& source_file : source_files_) { + auto name_length = static_cast(source_file.file_name.size()); + output.WriteValue(name_length); + output.Write(source_file.file_name.data(), name_length); + output.WriteValue(source_file.row_count); + } + PAIMON_UNIQUE_PTR bytes = MemorySegmentUtils::CopyToBytes( + output.Segments(), /*offset=*/0, static_cast(serialized_size), pool.get()); + return std::shared_ptr(std::move(bytes)); } } // namespace paimon diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta.h b/src/paimon/core/index/pk/primary_key_index_source_meta.h index 639f59852..dbfa485b2 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta.h +++ b/src/paimon/core/index/pk/primary_key_index_source_meta.h @@ -36,11 +36,11 @@ class IndexFileMeta; /// Ordered source data files covered by a source-backed primary-key index payload. /// -/// Wire format (version 1), byte-compatible with Java `PrimaryKeyIndexSourceMeta`: -/// big-endian int32 version, big-endian int32 data level (> 0), big-endian int32 source -/// file count (> 0), then per source file a Java `writeUTF` file name (uint16 big-endian -/// byte length + modified UTF-8 bytes) and a big-endian int64 row count. Trailing bytes -/// are rejected. +/// Wire format (version 1): big-endian int32 version, big-endian int32 data level (> 0), +/// big-endian int32 source file count (> 0), then per source file a uint16 big-endian byte +/// length, unchanged file name bytes, and a big-endian int64 row count. This matches Java +/// `writeUTF` for ASCII and non-null BMP UTF-8 file names. Java modified UTF-8 support for +/// supplementary code points requires a stream-level follow-up. Trailing bytes are rejected. class PrimaryKeyIndexSourceMeta { public: static constexpr int32_t VERSION = 1; @@ -53,7 +53,7 @@ class PrimaryKeyIndexSourceMeta { static Result Deserialize(const char* data, size_t length); - Result> Serialize(MemoryPool* pool) const; + Result> Serialize(const std::shared_ptr& pool) const; int32_t DataLevel() const { return data_level_; diff --git a/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp index 478589b12..44b83f5d3 100644 --- a/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp +++ b/src/paimon/core/index/pk/primary_key_index_source_meta_test.cpp @@ -20,6 +20,7 @@ #include "paimon/core/index/pk/primary_key_index_source_meta.h" #include +#include #include #include #include @@ -47,7 +48,7 @@ class PrimaryKeyIndexSourceMetaTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE( PrimaryKeyIndexSourceMeta meta, PrimaryKeyIndexSourceMeta::Create(data_level, std::move(source_files))); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, meta.Serialize(pool_.get())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, meta.Serialize(pool_)); return std::string(bytes->data(), bytes->size()); } @@ -80,14 +81,15 @@ TEST_F(PrimaryKeyIndexSourceMetaTest, SerializeMatchesGoldenBytes) { ASSERT_EQ(files, meta.SourceFiles()); } -TEST_F(PrimaryKeyIndexSourceMetaTest, RoundTripWithChineseNameAndLargeRowCount) { +TEST_F(PrimaryKeyIndexSourceMetaTest, RoundTripWithBmpNameAndLargeRowCount) { std::vector files; + // Java modified UTF-8 and standard UTF-8 use identical bytes for non-null BMP text. // Row count above 2^32 exercises the full big-endian int64 encoding. files.emplace_back("文件-0.parquet", (int64_t{1} << 40) + 7); files.emplace_back("data-1.parquet", 42); ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta meta, PrimaryKeyIndexSourceMeta::Create(5, files)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, meta.Serialize(pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr bytes, meta.Serialize(pool_)); ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta decoded, PrimaryKeyIndexSourceMeta::Deserialize(bytes->data(), bytes->size())); ASSERT_EQ(5, decoded.DataLevel()); @@ -161,12 +163,33 @@ TEST_F(PrimaryKeyIndexSourceMetaTest, CreateRejectsInvalidArguments) { ASSERT_NOK(PrimaryKeyIndexSourceMeta::Create(3, {{"a.parquet", -1}})); } +TEST_F(PrimaryKeyIndexSourceMetaTest, SerializeValidatesStringLengthAndMemoryPool) { + std::string maximum_name(std::numeric_limits::max(), 'a'); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta maximum_meta, + PrimaryKeyIndexSourceMeta::Create(1, {{maximum_name, 1}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr maximum_bytes, maximum_meta.Serialize(pool_)); + ASSERT_OK_AND_ASSIGN( + PrimaryKeyIndexSourceMeta maximum_decoded, + PrimaryKeyIndexSourceMeta::Deserialize(maximum_bytes->data(), maximum_bytes->size())); + ASSERT_EQ(maximum_meta.SourceFiles(), maximum_decoded.SourceFiles()); + + std::string oversized_name(static_cast(std::numeric_limits::max()) + 1, 'a'); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta oversized_meta, + PrimaryKeyIndexSourceMeta::Create(1, {{std::move(oversized_name), 1}})); + ASSERT_NOK_WITH_MSG(oversized_meta.Serialize(pool_), "too long for a 16-bit length"); + ASSERT_NOK_WITH_MSG(maximum_meta.Serialize(nullptr), "null memory pool"); +} + +TEST_F(PrimaryKeyIndexSourceMetaTest, DeserializeRejectsNullBuffer) { + ASSERT_NOK_WITH_MSG(PrimaryKeyIndexSourceMeta::Deserialize(nullptr, 0), "null buffer"); +} + TEST_F(PrimaryKeyIndexSourceMetaTest, FromIndexFileDecodesSourceMeta) { std::vector files; files.emplace_back("a.parquet", 100); ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta meta, PrimaryKeyIndexSourceMeta::Create(7, files)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr source_meta, meta.Serialize(pool_.get())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr source_meta, meta.Serialize(pool_)); std::shared_ptr index_meta = std::make_shared("index-payload", pool_.get()); GlobalIndexMeta global_index_meta(/*_row_range_start=*/0, /*_row_range_end=*/100, /*_index_field_id=*/1, /*_extra_field_ids=*/std::nullopt, diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp index 8e4c00521..d85045cac 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp @@ -61,9 +61,10 @@ class PkSortedBucketIndexStateTest : public ::testing::Test { int32_t field_id, const std::string& index_type, int32_t data_level, const std::vector& sources, int64_t total_row_count, int64_t row_range_start, int64_t row_range_end) const { - PrimaryKeyIndexSourceMeta source_meta = - PrimaryKeyIndexSourceMeta::Create(data_level, sources).value(); - std::shared_ptr source_meta_bytes = source_meta.Serialize(pool_.get()).value(); + EXPECT_OK_AND_ASSIGN(PrimaryKeyIndexSourceMeta source_meta, + PrimaryKeyIndexSourceMeta::Create(data_level, sources)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr source_meta_bytes, + source_meta.Serialize(pool_)); return MakePayloadWithSourceMetaBytes(field_id, index_type, total_row_count, row_range_start, row_range_end, source_meta_bytes); } diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp index 386f0a6f9..f919995b1 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp @@ -100,8 +100,7 @@ Result> PkSortedIndexFile::Build( } const GlobalIndexIOMeta& io_meta = io_metas[0]; - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr source_meta_bytes, - source_meta.Serialize(pool.get())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr source_meta_bytes, source_meta.Serialize(pool)); std::optional external_path; if (is_external_path) { external_path = io_meta.file_path; diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp index c529d5c18..90fda1a58 100644 --- a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp @@ -19,9 +19,13 @@ #include "paimon/core/table/source/primary_key_index_batch_scan.h" +#include +#include #include #include +#include +#include "fmt/format.h" #include "paimon/core/index/index_file_handler.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" #include "paimon/core/table/source/plan_impl.h" @@ -30,12 +34,29 @@ #include "paimon/core/table/source/snapshot/snapshot_reader.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" +#include "paimon/executor.h" #include "paimon/predicate/compound_predicate.h" #include "paimon/predicate/leaf_predicate.h" #include "paimon/predicate/predicate_builder.h" namespace paimon { namespace { +Result> CreateGlobalIndexExecutor(const CoreOptions& core_options) { + uint32_t thread_num = std::thread::hardware_concurrency(); + std::optional configured_thread_num = core_options.GetGlobalIndexThreadNum(); + if (configured_thread_num) { + if (configured_thread_num.value() <= 0) { + return Status::Invalid(fmt::format("invalid global index thread number {}", + configured_thread_num.value())); + } + thread_num = static_cast(configured_thread_num.value()); + } else if (thread_num == 0) { + thread_num = 1; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, CreateDefaultExecutor(thread_num)); + return executor; +} + /// Restricts a predicate to leaves over the indexed fields: an AND keeps its convertible /// children, an OR is only kept when every child is convertible, and everything else is /// dropped. A null return means no part of the predicate can use the index. @@ -252,10 +273,12 @@ Result> PrimaryKeyIndexBatchScan::CreatePlan() { PAIMON_ASSIGN_OR_RAISE(PrimaryKeySortedIndexScan::Plan index_plan, PrimaryKeySortedIndexScan::CreatePlan( snapshot_id, data_splits, scalar_definitions_, index_entries)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, + CreateGlobalIndexExecutor(core_options_)); PrimaryKeySortedIndexScan::ReaderFactory reader_factory = PrimaryKeySortedIndexScan::MakeReaderFactory( core_options_.GetFileSystem(), std::make_shared(path_factory_), - table_schema_, pool_); + table_schema_, pool_, executor); PAIMON_ASSIGN_OR_RAISE( PrimaryKeySortedIndexScan::EvaluatedPlan evaluated_plan, PrimaryKeySortedIndexScan::Evaluate(index_plan, table_schema_, index_predicate, diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp index 23684007a..8b56625d4 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -524,9 +524,10 @@ class FsGlobalIndexFileReader : public GlobalIndexFileReader { PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFactory( const std::shared_ptr& file_system, const std::shared_ptr& path_factories, - const std::shared_ptr& table_schema, const std::shared_ptr& pool) { + const std::shared_ptr& table_schema, const std::shared_ptr& pool, + const std::shared_ptr& executor) { auto file_reader = std::make_shared(file_system); - return [path_factories, table_schema, pool, file_reader]( + return [path_factories, table_schema, pool, file_reader, executor]( const FilePlan& file, const PrimaryKeyIndexDefinition& definition, const PkSortedIndexGroup& group) -> Result> { if (definition.GetFamily() != PrimaryKeyIndexDefinition::Family::BTREE) { @@ -560,7 +561,7 @@ PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFa ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); - return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool, executor); }; } diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.h b/src/paimon/core/table/source/primary_key_sorted_index_scan.h index e4838eb1c..a892f82ce 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.h +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.h @@ -41,6 +41,8 @@ #include "paimon/result.h" namespace paimon { +class Executor; + /// Plans and evaluates source-backed primary-key scalar index groups in file-local /// row-position space. /// @@ -179,7 +181,8 @@ class PrimaryKeySortedIndexScan { static ReaderFactory MakeReaderFactory( const std::shared_ptr& file_system, const std::shared_ptr& path_factories, - const std::shared_ptr& table_schema, const std::shared_ptr& pool); + const std::shared_ptr& table_schema, const std::shared_ptr& pool, + const std::shared_ptr& executor); }; } // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index 0e211d526..fdb6ffc7e 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -190,10 +190,9 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { /*partition_keys=*/std::vector(), /*primary_keys=*/std::vector{"id"}, options, /*comment=*/std::nullopt, /*time_millis=*/0); - Result definitions = - PrimaryKeyIndexDefinitions::Create(*table_schema_); - ASSERT_OK(definitions.status()); - definitions_ = definitions.value().ScalarDefinitions(); + ASSERT_OK_AND_ASSIGN(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema_)); + definitions_ = definitions.ScalarDefinitions(); ASSERT_EQ(definitions_.size(), 1); } @@ -249,9 +248,8 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { if (!deletion_files.empty()) { builder.WithDataDeletionFiles(deletion_files); } - Result> split = builder.Build(); - assert(split.ok()); - return split.value(); + EXPECT_OK_AND_ASSIGN(std::shared_ptr split, builder.Build()); + return split; } std::vector MakeEntries(const std::shared_ptr& payload) { @@ -314,18 +312,17 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { }; TEST_F(PrimaryKeySortedIndexScanTest, EqualNarrowsToSingleFileRange) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::shared_ptr split = MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, /*raw_convertible=*/true); // Value 10 sits at group ordinal 5, i.e. row 5 of a.parquet. - Result>> splits = PlanEvaluateConvert( - {split}, MakeEntries(payload.value()), PriceEqual(10), PayloadReaderFactory()); - ASSERT_OK(splits.status()); - ASSERT_EQ(splits.value().size(), 1); - auto indexed_split = std::dynamic_pointer_cast(splits.value()[0]); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + auto indexed_split = std::dynamic_pointer_cast(splits[0]); ASSERT_TRUE(indexed_split != nullptr); auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); ASSERT_TRUE(inner_split != nullptr); @@ -348,8 +345,7 @@ TEST_F(PrimaryKeySortedIndexScanTest, BuildRejectsDuplicateOrdinals) { } TEST_F(PrimaryKeySortedIndexScanTest, RangeSpansFileBoundary) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::shared_ptr split = MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, @@ -360,14 +356,14 @@ TEST_F(PrimaryKeySortedIndexScanTest, RangeSpansFileBoundary) { /*field_index=*/1, "price", FieldType::BIGINT, Literal(static_cast(190))); std::shared_ptr upper = PredicateBuilder::LessOrEqual( /*field_index=*/1, "price", FieldType::BIGINT, Literal(static_cast(210))); - Result> predicate = PredicateBuilder::And({lower, upper}); - ASSERT_OK(predicate.status()); - Result>> splits = PlanEvaluateConvert( - {split}, MakeEntries(payload.value()), predicate.value(), PayloadReaderFactory()); - ASSERT_OK(splits.status()); - ASSERT_EQ(splits.value().size(), 2); - auto indexed_a = std::dynamic_pointer_cast(splits.value()[0]); - auto indexed_b = std::dynamic_pointer_cast(splits.value()[1]); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({lower, upper})); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), predicate, PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 2); + auto indexed_a = std::dynamic_pointer_cast(splits[0]); + auto indexed_b = std::dynamic_pointer_cast(splits[1]); ASSERT_TRUE(indexed_a != nullptr); ASSERT_TRUE(indexed_b != nullptr); ASSERT_EQ(indexed_a->RowRanges().size(), 1); @@ -379,75 +375,69 @@ TEST_F(PrimaryKeySortedIndexScanTest, RangeSpansFileBoundary) { } TEST_F(PrimaryKeySortedIndexScanTest, EmptyResultOmitsAllFiles) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::shared_ptr split = MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, /*raw_convertible=*/true); // All indexed values are even, so 11 matches nothing. - Result>> splits = PlanEvaluateConvert( - {split}, MakeEntries(payload.value()), PriceEqual(11), PayloadReaderFactory()); - ASSERT_OK(splits.status()); - ASSERT_TRUE(splits.value().empty()); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(11), PayloadReaderFactory())); + ASSERT_TRUE(splits.empty()); } TEST_F(PrimaryKeySortedIndexScanTest, UnindexedFieldPredicateFallsBack) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::shared_ptr split = MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, /*raw_convertible=*/true); std::shared_ptr predicate = PredicateBuilder::Equal( /*field_index=*/2, "status", FieldType::STRING, Literal(FieldType::STRING, "hit", 3)); - Result>> splits = PlanEvaluateConvert( - {split}, MakeEntries(payload.value()), predicate, PayloadReaderFactory()); - ASSERT_OK(splits.status()); - ASSERT_EQ(1, splits.value().size()); - ASSERT_EQ(split, splits.value()[0]); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), predicate, PayloadReaderFactory())); + ASSERT_EQ(1, splits.size()); + ASSERT_EQ(split, splits[0]); } TEST_F(PrimaryKeySortedIndexScanTest, UncoveredFileFallsBackOthersNarrow) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::shared_ptr split = MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact()), MakeDataFile("c.parquet", 50, 0, FileSource::Append())}, /*raw_convertible=*/true); - Result>> splits = PlanEvaluateConvert( - {split}, MakeEntries(payload.value()), PriceEqual(10), PayloadReaderFactory()); - ASSERT_OK(splits.status()); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); // a.parquet narrows to an indexed split, b.parquet is omitted, c.parquet has no // coverage and keeps a normal single-file scan. - ASSERT_EQ(splits.value().size(), 2); - auto indexed_split = std::dynamic_pointer_cast(splits.value()[0]); + ASSERT_EQ(splits.size(), 2); + auto indexed_split = std::dynamic_pointer_cast(splits[0]); ASSERT_TRUE(indexed_split != nullptr); - auto fallback_split = std::dynamic_pointer_cast(splits.value()[1]); + auto fallback_split = std::dynamic_pointer_cast(splits[1]); ASSERT_TRUE(fallback_split != nullptr); ASSERT_EQ(fallback_split->DataFiles().size(), 1); ASSERT_EQ(fallback_split->DataFiles()[0]->file_name, "c.parquet"); } TEST_F(PrimaryKeySortedIndexScanTest, NonRawConvertibleSplitPreserved) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::shared_ptr split = MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, /*raw_convertible=*/false); - Result>> splits = PlanEvaluateConvert( - {split}, MakeEntries(payload.value()), PriceEqual(10), PayloadReaderFactory()); - ASSERT_OK(splits.status()); - ASSERT_EQ(splits.value().size(), 1); - ASSERT_EQ(splits.value()[0].get(), split.get()); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + ASSERT_EQ(splits[0].get(), split.get()); } TEST_F(PrimaryKeySortedIndexScanTest, InvalidRowRangePayloadFallsBack) { - Result> payload_result = BuildPayload(); - ASSERT_OK(payload_result.status()); - const std::shared_ptr& payload = payload_result.value(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); // Rebuild the payload metadata with a row range end beyond the source rows: the group // validation must reject it and every file keeps a normal scan. const GlobalIndexMeta& meta = payload->GetGlobalIndexMeta().value(); @@ -460,16 +450,15 @@ TEST_F(PrimaryKeySortedIndexScanTest, InvalidRowRangePayloadFallsBack) { MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, /*raw_convertible=*/true); - Result>> splits = PlanEvaluateConvert( - {split}, MakeEntries(broken_payload), PriceEqual(10), PayloadReaderFactory()); - ASSERT_OK(splits.status()); - ASSERT_EQ(1, splits.value().size()); - ASSERT_EQ(split, splits.value()[0]); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(broken_payload), PriceEqual(10), + PayloadReaderFactory())); + ASSERT_EQ(1, splits.size()); + ASSERT_EQ(split, splits[0]); } TEST_F(PrimaryKeySortedIndexScanTest, OutOfRangePositionsFailAllCoveredFiles) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::shared_ptr split = MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, @@ -483,30 +472,28 @@ TEST_F(PrimaryKeySortedIndexScanTest, OutOfRangePositionsFailAllCoveredFiles) { const PkSortedIndexGroup& group) -> Result> { return std::make_shared(poisoned); }; - Result>> splits = - PlanEvaluateConvert({split}, MakeEntries(payload.value()), PriceEqual(10), stub_factory); - ASSERT_OK(splits.status()); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), stub_factory)); // Both covered files fall back together, preserving the planner's original bin packing. - ASSERT_EQ(1, splits.value().size()); - ASSERT_EQ(split, splits.value()[0]); + ASSERT_EQ(1, splits.size()); + ASSERT_EQ(split, splits[0]); } TEST_F(PrimaryKeySortedIndexScanTest, OverFragmentedResultFallsBack) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); // One data file, 20000 rows; every second row selected produces > 4096 ranges. std::vector source_files = {{"big.parquet", 20000}}; std::shared_ptr split = MakeSplit( {MakeDataFile("big.parquet", 20000, 5, FileSource::Compact())}, /*raw_convertible=*/true); - Result> source_meta_bytes = [&]() -> Result> { - PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexSourceMeta source_meta, - PrimaryKeyIndexSourceMeta::Create(5, source_files)); - return source_meta.Serialize(pool_.get()); - }(); - ASSERT_OK(source_meta_bytes.status()); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr source_meta_bytes, ([&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexSourceMeta source_meta, + PrimaryKeyIndexSourceMeta::Create(5, source_files)); + return source_meta.Serialize(pool_); + }())); auto big_payload = std::make_shared( "btree", "big-index-file", /*file_size=*/1, /*row_count=*/20000, std::nullopt, std::nullopt, - GlobalIndexMeta(0, 19999, kPriceFieldId, std::nullopt, nullptr, source_meta_bytes.value())); + GlobalIndexMeta(0, 19999, kPriceFieldId, std::nullopt, nullptr, source_meta_bytes)); RoaringBitmap64 fragmented; for (int64_t i = 0; i < 20000; i += 2) { fragmented.Add(i); @@ -518,26 +505,25 @@ TEST_F(PrimaryKeySortedIndexScanTest, OverFragmentedResultFallsBack) { const PkSortedIndexGroup& group) -> Result> { return std::make_shared(fragmented); }; - Result>> splits = - PlanEvaluateConvert({split}, MakeEntries(big_payload), PriceEqual(10), stub_factory); - ASSERT_OK(splits.status()); - ASSERT_EQ(splits.value().size(), 1); - ASSERT_TRUE(std::dynamic_pointer_cast(splits.value()[0]) == nullptr); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(big_payload), PriceEqual(10), stub_factory)); + ASSERT_EQ(splits.size(), 1); + ASSERT_TRUE(std::dynamic_pointer_cast(splits[0]) == nullptr); } TEST_F(PrimaryKeySortedIndexScanTest, DeletionFileStaysAlignedWithIndexedFile) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); DeletionFile deletion_file("dv-a", /*offset=*/0, /*length=*/16, /*cardinality=*/1); std::shared_ptr split = MakeSplit( {MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, /*raw_convertible=*/true, {std::optional(deletion_file), std::nullopt}); - Result>> splits = PlanEvaluateConvert( - {split}, MakeEntries(payload.value()), PriceEqual(10), PayloadReaderFactory()); - ASSERT_OK(splits.status()); - ASSERT_EQ(splits.value().size(), 1); - auto indexed_split = std::dynamic_pointer_cast(splits.value()[0]); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + auto indexed_split = std::dynamic_pointer_cast(splits[0]); ASSERT_TRUE(indexed_split != nullptr); auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); ASSERT_TRUE(inner_split != nullptr); @@ -547,17 +533,15 @@ TEST_F(PrimaryKeySortedIndexScanTest, DeletionFileStaysAlignedWithIndexedFile) { } TEST_F(PrimaryKeySortedIndexScanTest, SnapshotMismatchIsRejected) { - Result> payload = BuildPayload(); - ASSERT_OK(payload.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::vector> files = { MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact())}; DataSplitImpl::Builder builder(BinaryRow::EmptyRow(), /*bucket=*/0, base_path_, std::move(files)); builder.WithSnapshot(kSnapshotId + 1).IsStreaming(false).RawConvertible(true); - Result> split = builder.Build(); - ASSERT_OK(split.status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, builder.Build()); Result plan = PrimaryKeySortedIndexScan::CreatePlan( - kSnapshotId, {split.value()}, definitions_, MakeEntries(payload.value())); + kSnapshotId, {split}, definitions_, MakeEntries(payload)); ASSERT_NOK(plan.status()); } From b02dd5f76aeba11dbaeef16af476ca57e6acb7e7 Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Thu, 13 Aug 2026 08:24:13 -0400 Subject: [PATCH 09/14] perf(pk-index): avoid unused scan executor --- .../core/table/source/primary_key_index_batch_scan.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp index 90fda1a58..9425c1a06 100644 --- a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp @@ -19,6 +19,7 @@ #include "paimon/core/table/source/primary_key_index_batch_scan.h" +#include #include #include #include @@ -273,6 +274,12 @@ Result> PrimaryKeyIndexBatchScan::CreatePlan() { PAIMON_ASSIGN_OR_RAISE(PrimaryKeySortedIndexScan::Plan index_plan, PrimaryKeySortedIndexScan::CreatePlan( snapshot_id, data_splits, scalar_definitions_, index_entries)); + bool has_index_group = std::any_of( + index_plan.Files().begin(), index_plan.Files().end(), + [](const PrimaryKeySortedIndexScan::FilePlan& file) { return !file.Groups().empty(); }); + if (!has_index_group) { + return data_plan; + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, CreateGlobalIndexExecutor(core_options_)); PrimaryKeySortedIndexScan::ReaderFactory reader_factory = From 5e36b66e072acbd7f5d3f61a93966831d850d8ae Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Sun, 16 Aug 2026 03:58:25 -0400 Subject: [PATCH 10/14] fix(pk-index): address review feedback --- src/paimon/CMakeLists.txt | 1 + .../global_index_evaluator_impl.cpp | 105 +++- .../global_index_evaluator_impl.h | 5 + .../global_index_evaluator_impl_test.cpp | 293 ++++++++++ .../pk/primary_key_index_definitions.cpp | 7 +- .../index/pk/primary_key_index_definitions.h | 4 + .../pksorted/pk_sorted_bucket_index_state.cpp | 8 +- .../pksorted/pk_sorted_bucket_index_state.h | 6 +- .../pk_sorted_bucket_index_state_test.cpp | 16 +- .../index/pksorted/pk_sorted_index_file.cpp | 5 +- .../index/pksorted/pk_sorted_index_group.cpp | 14 +- .../index/pksorted/pk_sorted_index_group.h | 7 +- .../append_only_file_store_write.cpp | 4 +- .../core/operation/raw_file_split_read.cpp | 47 +- .../core/operation/raw_file_split_read.h | 10 +- .../operation/raw_file_split_read_test.cpp | 17 + .../table/source/key_value_table_read.cpp | 32 +- .../source/primary_key_index_batch_scan.cpp | 160 +----- .../primary_key_sorted_index_result.cpp | 2 +- .../source/primary_key_sorted_index_scan.cpp | 72 +-- .../primary_key_sorted_index_scan_test.cpp | 75 ++- test/inte/CMakeLists.txt | 7 + .../primary_key_sorted_index_inte_test.cpp | 524 ++++++++++++++++++ .../orc/pk_btree_e2e.db/pk_btree_e2e/README | 24 + ...5ee15f36-7a45-4651-8c81-a55e49dbf9ff-0.orc | Bin 0 -> 757 bytes ...c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-0.orc | Bin 0 -> 854 bytes ...c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-1.orc | Bin 0 -> 874 bytes ...c917f42c-c01c-4d16-9c2a-cb89c60d8d6a-0.orc | Bin 0 -> 764 bytes ...e165b892-027d-4e7b-9750-3226cd0410af-0.orc | Bin 0 -> 1543 bytes ...e165b892-027d-4e7b-9750-3226cd0410af-1.orc | Bin 0 -> 909 bytes ...ed9149c4-c451-4780-a2cd-fe8a74ee2558-0.orc | Bin 0 -> 732 bytes ...f663e836-b9cb-45fc-b513-266c4b33026f-0.orc | Bin 0 -> 854 bytes ...f663e836-b9cb-45fc-b513-266c4b33026f-1.orc | Bin 0 -> 874 bytes ...dex-2e50e625-15ac-4994-a320-064d8e34d028-0 | Bin 0 -> 4083 bytes ...dex-5890ac3e-74e0-4d3c-bddd-542207d3a13f-0 | 1 + ...dex-7d01de51-d440-42c2-a2ac-f8b89d8e9d36-0 | Bin 0 -> 33 bytes ...dex-a3ae7e50-6220-4663-89de-4b9b31f41a39-0 | Bin 0 -> 88 bytes ...dex-c940705f-8d27-40f3-a9ba-a896eeecd29a-0 | Bin 0 -> 4096 bytes ...est-456ef7ae-b9e5-4e2a-bc33-b6afade33eac-0 | Bin 0 -> 1593 bytes ...est-9dbada10-b497-4ef7-bf69-a3bdb5959a21-0 | Bin 0 -> 1442 bytes ...est-cbbe0867-55db-48b3-a617-01a8b5ea66e8-0 | Bin 0 -> 1496 bytes ...est-5db2d0f1-0070-4dd6-b135-b62ce82c1502-0 | Bin 0 -> 2161 bytes ...est-8175e923-c512-4513-9325-bbf4a875b2d9-0 | Bin 0 -> 2224 bytes ...est-999f9f03-1b9c-4967-bda2-441ecce6bc77-0 | Bin 0 -> 2104 bytes ...est-c08df75e-4def-48df-b351-b5fe73087467-0 | Bin 0 -> 2135 bytes ...est-d102f891-c912-4cff-92f7-f5765f8d2248-0 | Bin 0 -> 2442 bytes ...ist-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-0 | Bin 0 -> 1184 bytes ...ist-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-1 | Bin 0 -> 1117 bytes ...ist-619f76ac-a10a-449b-a384-6ad6d69d1912-0 | Bin 0 -> 1148 bytes ...ist-619f76ac-a10a-449b-a384-6ad6d69d1912-1 | Bin 0 -> 1117 bytes ...ist-61aa9e49-d705-426e-adc6-4929d7fa79b5-0 | Bin 0 -> 1006 bytes ...ist-61aa9e49-d705-426e-adc6-4929d7fa79b5-1 | Bin 0 -> 1113 bytes ...ist-77c3cc85-9a12-4773-aedd-0de9beb4230a-0 | Bin 0 -> 1218 bytes ...ist-77c3cc85-9a12-4773-aedd-0de9beb4230a-1 | Bin 0 -> 1114 bytes ...ist-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-0 | Bin 0 -> 1113 bytes ...ist-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-1 | Bin 0 -> 1115 bytes .../pk_btree_e2e/schema/schema-0 | 32 ++ .../pk_btree_e2e/snapshot/EARLIEST | 1 + .../pk_btree_e2e/snapshot/LATEST | 1 + .../pk_btree_e2e/snapshot/snapshot-1 | 17 + .../pk_btree_e2e/snapshot/snapshot-2 | 18 + .../pk_btree_e2e/snapshot/snapshot-3 | 18 + .../pk_btree_e2e/snapshot/snapshot-4 | 18 + .../pk_btree_e2e/snapshot/snapshot-5 | 18 + .../pk_btree_partitioned_e2e/README | 21 + .../branch/branch-fallback/schema/schema-0 | 36 ++ .../branch-fallback/snapshot/snapshot-2 | 18 + .../branch-fallback/tag/tag-fallback-base | 18 + ...dex-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-0 | 1 + ...dex-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-1 | 1 + ...dex-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-0 | Bin 0 -> 247 bytes ...dex-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-1 | Bin 0 -> 247 bytes ...dex-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-2 | Bin 0 -> 235 bytes ...dex-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-3 | Bin 0 -> 247 bytes ...dex-986b5fda-9721-4e64-9a1a-36b9bf2ad357-0 | Bin 0 -> 88 bytes ...dex-b441165c-929b-4f52-b27c-a9598db52b36-0 | Bin 0 -> 235 bytes ...dex-b441165c-929b-4f52-b27c-a9598db52b36-1 | Bin 0 -> 247 bytes ...dex-b441165c-929b-4f52-b27c-a9598db52b36-2 | Bin 0 -> 235 bytes ...dex-b441165c-929b-4f52-b27c-a9598db52b36-3 | Bin 0 -> 247 bytes ...dex-e889a0de-d6c3-479e-967f-2e62654d3790-0 | Bin 0 -> 31 bytes ...dex-e889a0de-d6c3-479e-967f-2e62654d3790-1 | Bin 0 -> 31 bytes ...est-2811d607-fabd-43f1-bc76-82a2051280a0-0 | Bin 0 -> 1549 bytes ...est-55558948-e419-4c96-912a-f7d4b1d25af6-0 | Bin 0 -> 1644 bytes ...est-cb21ca4f-0678-48b5-b2ba-f797e7dedab4-0 | Bin 0 -> 1704 bytes ...est-1b596e7e-4a38-4963-8e62-1c50fb11c73e-0 | Bin 0 -> 2592 bytes ...est-24630e00-34bd-45e8-9ffc-e1b061217359-0 | Bin 0 -> 2251 bytes ...est-8f4fff72-e24d-4ac8-b2fe-a39b2ba3d528-0 | Bin 0 -> 2147 bytes ...est-b3d6bc4a-6665-49ea-9d2e-0f2ac5fd142c-0 | Bin 0 -> 2107 bytes ...est-e2a0a966-17b2-4269-9362-2bde27472a88-0 | Bin 0 -> 2407 bytes ...ist-507ec8c8-79e9-40d0-a05d-4a402f7726a6-0 | Bin 0 -> 1006 bytes ...ist-507ec8c8-79e9-40d0-a05d-4a402f7726a6-1 | Bin 0 -> 1118 bytes ...ist-953de685-b6f6-427e-bd84-42fe41e9ac1f-0 | Bin 0 -> 1196 bytes ...ist-953de685-b6f6-427e-bd84-42fe41e9ac1f-1 | Bin 0 -> 1122 bytes ...ist-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-0 | Bin 0 -> 1234 bytes ...ist-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-1 | Bin 0 -> 1126 bytes ...ist-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0 | Bin 0 -> 1118 bytes ...ist-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1 | Bin 0 -> 1123 bytes ...ist-cf15e41d-848f-42d5-b0af-d75ffef9386f-0 | Bin 0 -> 1159 bytes ...ist-cf15e41d-848f-42d5-b0af-d75ffef9386f-1 | Bin 0 -> 1123 bytes ...0038846e-e990-4367-a488-f390312bae65-0.orc | Bin 0 -> 736 bytes ...0d30be46-8b96-4020-b2fc-ec814be128c2-0.orc | Bin 0 -> 736 bytes ...76bfec84-0851-4a14-8c8e-5e9f54c18ef2-0.orc | Bin 0 -> 916 bytes ...a6c5a536-5e45-4d9b-baae-742f93ee670d-0.orc | Bin 0 -> 924 bytes ...a6c5a536-5e45-4d9b-baae-742f93ee670d-1.orc | Bin 0 -> 736 bytes ...e32042c3-9e41-4cd7-8bcb-59e0944b3803-0.orc | Bin 0 -> 916 bytes ...1245ceb5-2933-4397-89f9-5363b9a46498-0.orc | Bin 0 -> 946 bytes ...dc40f35c-9fff-40a3-a08b-38898a7538a9-0.orc | Bin 0 -> 946 bytes ...f818ca94-495e-4146-a8ad-35a4474f5058-0.orc | Bin 0 -> 946 bytes ...3940b910-9cc6-44bd-8a34-ee00e35f4977-0.orc | Bin 0 -> 755 bytes ...7e663126-95fd-402d-b971-80372baa3fe7-0.orc | Bin 0 -> 918 bytes ...835df91b-8a15-4ee4-b118-4a4f5429c2f1-0.orc | Bin 0 -> 918 bytes ...91617322-2bb2-4ad1-a445-e1dd850d2139-0.orc | Bin 0 -> 735 bytes ...a2e76792-9576-4276-8bc0-8a76cbeb4f9c-0.orc | Bin 0 -> 977 bytes ...dcc6ed0d-38b1-4447-b3e6-5b1788fc948a-0.orc | Bin 0 -> 755 bytes ...2a0e2e33-9592-4be7-9d68-d7f57c9d735c-0.orc | Bin 0 -> 946 bytes ...c1dcd608-70b5-4d7f-afd2-676c97fb9fac-0.orc | Bin 0 -> 946 bytes ...d9aabc21-f287-48ae-baea-5b6d70640f0d-0.orc | Bin 0 -> 946 bytes .../pk_btree_partitioned_e2e/schema/schema-0 | 36 ++ .../snapshot/EARLIEST | 1 + .../pk_btree_partitioned_e2e/snapshot/LATEST | 1 + .../snapshot/snapshot-1 | 17 + .../snapshot/snapshot-2 | 18 + .../snapshot/snapshot-3 | 18 + .../snapshot/snapshot-4 | 18 + .../snapshot/snapshot-5 | 18 + .../tag/tag-fallback-base | 18 + .../pk_btree_e2e.db/pk_btree_e2e/README | 24 + ...e41e-5153-4d29-8a48-dbbdf015f77a-0.parquet | Bin 0 -> 1322 bytes ...c2d2-86bc-4305-8ddc-3d7587b8e8ac-0.parquet | Bin 0 -> 1269 bytes ...b187-f60c-439b-841c-b0fa0a78da4f-0.parquet | Bin 0 -> 1277 bytes ...5c8e-2c9f-46a7-ad5d-4423f7cc5d59-0.parquet | Bin 0 -> 6666 bytes ...5c8e-2c9f-46a7-ad5d-4423f7cc5d59-1.parquet | Bin 0 -> 6570 bytes ...2a1d-4256-4c89-a3e9-51bedc74f336-0.parquet | Bin 0 -> 6666 bytes ...2a1d-4256-4c89-a3e9-51bedc74f336-1.parquet | Bin 0 -> 6570 bytes ...04c3-09a9-4c6e-b16a-9081d4a3faa7-0.parquet | Bin 0 -> 6392 bytes ...04c3-09a9-4c6e-b16a-9081d4a3faa7-1.parquet | Bin 0 -> 6570 bytes ...04c3-09a9-4c6e-b16a-9081d4a3faa7-2.parquet | Bin 0 -> 1322 bytes ...dex-1e092fe2-2c76-4682-bfec-50ebecd3957a-0 | Bin 0 -> 4096 bytes ...dex-360cfb9c-6646-44cb-aec5-d156a379b4d2-0 | Bin 0 -> 4083 bytes ...dex-5561631d-80ef-45bc-8b87-3d1a2876816d-0 | Bin 0 -> 33 bytes ...dex-806143e3-9271-4934-9ca5-636a815c46a0-0 | Bin 0 -> 88 bytes ...dex-a353b7ca-671b-4423-bff5-fd2888db7536-0 | 1 + ...est-22adae88-c6c0-4120-9d67-4961b05d16a6-0 | Bin 0 -> 1445 bytes ...est-c09d3b21-6083-4881-a6f6-b0ecc37dac22-0 | Bin 0 -> 1589 bytes ...est-d52df85e-6f91-497a-8edd-df92bcc3e772-0 | Bin 0 -> 1507 bytes ...est-425d391e-cf9c-4770-9241-09facc794f72-0 | Bin 0 -> 2169 bytes ...est-5b55486f-5a96-408a-9079-c51b68c5dec9-0 | Bin 0 -> 2261 bytes ...est-97c08d9a-9e64-4d25-8e37-3350e3eba2aa-0 | Bin 0 -> 2108 bytes ...est-a01531dd-94c8-4b58-b844-b0e1f26a464c-0 | Bin 0 -> 2425 bytes ...est-dc356a38-74d8-41b5-a7b8-270860bf71ad-0 | Bin 0 -> 2137 bytes ...ist-56d505a4-531c-494d-a72f-4adf49a50905-0 | Bin 0 -> 1110 bytes ...ist-56d505a4-531c-494d-a72f-4adf49a50905-1 | Bin 0 -> 1112 bytes ...ist-5c3ee90f-8062-405c-a97b-b5703fcc06c6-0 | Bin 0 -> 1186 bytes ...ist-5c3ee90f-8062-405c-a97b-b5703fcc06c6-1 | Bin 0 -> 1117 bytes ...ist-b6b5269a-5837-44e2-94c5-5033b77ca6de-0 | Bin 0 -> 1006 bytes ...ist-b6b5269a-5837-44e2-94c5-5033b77ca6de-1 | Bin 0 -> 1110 bytes ...ist-d5365195-730d-4f66-844c-7ef48b6df2a6-0 | Bin 0 -> 1149 bytes ...ist-d5365195-730d-4f66-844c-7ef48b6df2a6-1 | Bin 0 -> 1117 bytes ...ist-fba7550c-51f9-44c1-af46-e55fb9b308b1-0 | Bin 0 -> 1219 bytes ...ist-fba7550c-51f9-44c1-af46-e55fb9b308b1-1 | Bin 0 -> 1117 bytes .../pk_btree_e2e/schema/schema-0 | 32 ++ .../pk_btree_e2e/snapshot/EARLIEST | 1 + .../pk_btree_e2e/snapshot/LATEST | 1 + .../pk_btree_e2e/snapshot/snapshot-1 | 17 + .../pk_btree_e2e/snapshot/snapshot-2 | 18 + .../pk_btree_e2e/snapshot/snapshot-3 | 18 + .../pk_btree_e2e/snapshot/snapshot-4 | 18 + .../pk_btree_e2e/snapshot/snapshot-5 | 18 + .../pk_btree_partitioned_e2e/README | 21 + .../branch/branch-fallback/schema/schema-0 | 36 ++ .../branch-fallback/snapshot/snapshot-2 | 18 + .../branch-fallback/tag/tag-fallback-base | 18 + ...dex-3b03f22c-33cf-4746-a2b5-d4652476121b-0 | Bin 0 -> 31 bytes ...dex-3b03f22c-33cf-4746-a2b5-d4652476121b-1 | Bin 0 -> 31 bytes ...dex-523fefd0-769f-46f8-8b58-957065ad7e20-0 | 1 + ...dex-523fefd0-769f-46f8-8b58-957065ad7e20-1 | 1 + ...dex-622c2733-874d-4ebb-b076-500a21ad0432-0 | Bin 0 -> 88 bytes ...dex-7f29a66a-2114-4c06-8a28-b10f3cdce826-0 | Bin 0 -> 235 bytes ...dex-7f29a66a-2114-4c06-8a28-b10f3cdce826-1 | Bin 0 -> 247 bytes ...dex-7f29a66a-2114-4c06-8a28-b10f3cdce826-2 | Bin 0 -> 235 bytes ...dex-7f29a66a-2114-4c06-8a28-b10f3cdce826-3 | Bin 0 -> 247 bytes ...dex-e5696b70-4034-43bd-a42c-43e87e0a72b7-0 | Bin 0 -> 247 bytes ...dex-e5696b70-4034-43bd-a42c-43e87e0a72b7-1 | Bin 0 -> 247 bytes ...dex-e5696b70-4034-43bd-a42c-43e87e0a72b7-2 | Bin 0 -> 235 bytes ...dex-e5696b70-4034-43bd-a42c-43e87e0a72b7-3 | Bin 0 -> 247 bytes ...est-11be7772-670d-4dc1-adbb-fd8413f2779d-0 | Bin 0 -> 1647 bytes ...est-74eb4325-f527-47fc-aeea-25b0928190f7-0 | Bin 0 -> 1548 bytes ...est-a9b54c7b-7468-4a64-97ee-32adae28681e-0 | Bin 0 -> 1694 bytes ...est-8727f0fe-6af1-4c2f-8560-c38dd0a81ec0-0 | Bin 0 -> 2149 bytes ...est-bd400d22-3691-47bb-8262-753eb1a8c2c4-0 | Bin 0 -> 2254 bytes ...est-dd531b47-e547-4b27-b799-09842a4b026d-0 | Bin 0 -> 2106 bytes ...est-e6ecca13-a405-472b-b1a4-77de887ae406-0 | Bin 0 -> 2591 bytes ...est-f8dc9dce-972e-41a4-b6dc-a8eae92847f4-0 | Bin 0 -> 2408 bytes ...ist-1943462b-63ba-44d3-8893-0f2557f3b1a9-0 | Bin 0 -> 1196 bytes ...ist-1943462b-63ba-44d3-8893-0f2557f3b1a9-1 | Bin 0 -> 1123 bytes ...ist-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0 | Bin 0 -> 1119 bytes ...ist-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1 | Bin 0 -> 1123 bytes ...ist-692295fa-2039-44ff-b207-b0df5c7f715b-0 | Bin 0 -> 1158 bytes ...ist-692295fa-2039-44ff-b207-b0df5c7f715b-1 | Bin 0 -> 1123 bytes ...ist-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-0 | Bin 0 -> 1006 bytes ...ist-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-1 | Bin 0 -> 1119 bytes ...ist-f1757963-61f0-4136-bae6-cf52e0b2f42c-0 | Bin 0 -> 1234 bytes ...ist-f1757963-61f0-4136-bae6-cf52e0b2f42c-1 | Bin 0 -> 1126 bytes ...504c-fd4a-4ef7-8f63-724e727201e9-0.parquet | Bin 0 -> 1409 bytes ...de13-5159-4129-916b-ec43bbf7c968-0.parquet | Bin 0 -> 1878 bytes ...de13-5159-4129-916b-ec43bbf7c968-1.parquet | Bin 0 -> 1409 bytes ...b55f-03c4-4cdf-829a-1cd0c9d670ce-0.parquet | Bin 0 -> 1411 bytes ...efe8-01bf-4123-a9f6-e9505701ffba-0.parquet | Bin 0 -> 1885 bytes ...ddb8-fcd1-4298-a4a8-4778ab7aa7a6-0.parquet | Bin 0 -> 1885 bytes ...1bb7-3867-4575-a7f3-a649acff590b-0.parquet | Bin 0 -> 1932 bytes ...0aa8-9cf7-409d-a011-f7d6a0cad4a8-0.parquet | Bin 0 -> 1932 bytes ...9d3b-bb1f-4afe-a6b3-8cc85490af74-0.parquet | Bin 0 -> 1932 bytes ...39a0-6e5d-44a8-a6b1-e43e09d661b5-0.parquet | Bin 0 -> 1432 bytes ...6d53-21db-4d51-83e6-ea39269e80ca-0.parquet | Bin 0 -> 1885 bytes ...0233-1c62-46c7-89f1-6093f22e9eee-0.parquet | Bin 0 -> 1432 bytes ...c24c-7e5d-4179-9bbe-add3159334d0-0.parquet | Bin 0 -> 1409 bytes ...d079-dfcb-46f0-8b5b-ef1e35e78711-0.parquet | Bin 0 -> 1885 bytes ...daa8-a1c3-4b09-b4a4-9f58a1accd7f-0.parquet | Bin 0 -> 1914 bytes ...1d2e-69d0-4f8f-87f1-110f34c228d7-0.parquet | Bin 0 -> 1932 bytes ...d75f-edd3-4307-9473-f5826baf5d96-0.parquet | Bin 0 -> 1932 bytes ...e01a-3620-40bf-9da7-e20669696822-0.parquet | Bin 0 -> 1932 bytes .../pk_btree_partitioned_e2e/schema/schema-0 | 36 ++ .../snapshot/EARLIEST | 1 + .../pk_btree_partitioned_e2e/snapshot/LATEST | 1 + .../snapshot/snapshot-1 | 17 + .../snapshot/snapshot-2 | 18 + .../snapshot/snapshot-3 | 18 + .../snapshot/snapshot-4 | 18 + .../snapshot/snapshot-5 | 18 + .../tag/tag-fallback-base | 18 + 230 files changed, 1924 insertions(+), 273 deletions(-) create mode 100644 src/paimon/core/global_index/global_index_evaluator_impl_test.cpp create mode 100644 test/inte/primary_key_sorted_index_inte_test.cpp create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/README create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-5ee15f36-7a45-4651-8c81-a55e49dbf9ff-0.orc create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-0.orc create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-1.orc create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c917f42c-c01c-4d16-9c2a-cb89c60d8d6a-0.orc create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-0.orc create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-1.orc create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-ed9149c4-c451-4780-a2cd-fe8a74ee2558-0.orc create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-0.orc create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-1.orc create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-2e50e625-15ac-4994-a320-064d8e34d028-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-5890ac3e-74e0-4d3c-bddd-542207d3a13f-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-7d01de51-d440-42c2-a2ac-f8b89d8e9d36-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-a3ae7e50-6220-4663-89de-4b9b31f41a39-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-c940705f-8d27-40f3-a9ba-a896eeecd29a-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-456ef7ae-b9e5-4e2a-bc33-b6afade33eac-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-9dbada10-b497-4ef7-bf69-a3bdb5959a21-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-cbbe0867-55db-48b3-a617-01a8b5ea66e8-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-5db2d0f1-0070-4dd6-b135-b62ce82c1502-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-8175e923-c512-4513-9325-bbf4a875b2d9-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-999f9f03-1b9c-4967-bda2-441ecce6bc77-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-c08df75e-4def-48df-b351-b5fe73087467-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-d102f891-c912-4cff-92f7-f5765f8d2248-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-1 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-1 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-1 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-1 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-1 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 create mode 100644 test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3c4a320b-a1e7-4e21-9480-a7c62d1b97d7-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-2 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-3 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-986b5fda-9721-4e64-9a1a-36b9bf2ad357-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-2 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-3 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-55558948-e419-4c96-912a-f7d4b1d25af6-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-cb21ca4f-0678-48b5-b2ba-f797e7dedab4-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-1b596e7e-4a38-4963-8e62-1c50fb11c73e-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-24630e00-34bd-45e8-9ffc-e1b061217359-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8f4fff72-e24d-4ac8-b2fe-a39b2ba3d528-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-b3d6bc4a-6665-49ea-9d2e-0f2ac5fd142c-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e2a0a966-17b2-4269-9362-2bde27472a88-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0038846e-e990-4367-a488-f390312bae65-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0d30be46-8b96-4020-b2fc-ec814be128c2-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-76bfec84-0851-4a14-8c8e-5e9f54c18ef2-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-a6c5a536-5e45-4d9b-baae-742f93ee670d-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-a6c5a536-5e45-4d9b-baae-742f93ee670d-1.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-e32042c3-9e41-4cd7-8bcb-59e0944b3803-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-1245ceb5-2933-4397-89f9-5363b9a46498-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-dc40f35c-9fff-40a3-a08b-38898a7538a9-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-f818ca94-495e-4146-a8ad-35a4474f5058-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-3940b910-9cc6-44bd-8a34-ee00e35f4977-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-7e663126-95fd-402d-b971-80372baa3fe7-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-835df91b-8a15-4ee4-b118-4a4f5429c2f1-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-91617322-2bb2-4ad1-a445-e1dd850d2139-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-a2e76792-9576-4276-8bc0-8a76cbeb4f9c-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-dcc6ed0d-38b1-4447-b3e6-5b1788fc948a-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-2a0e2e33-9592-4be7-9d68-d7f57c9d735c-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1dcd608-70b5-4d7f-afd2-676c97fb9fac-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-d9aabc21-f287-48ae-baea-5b6d70640f0d-0.orc create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 create mode 100644 test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/README create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-0caae41e-5153-4d29-8a48-dbbdf015f77a-0.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-14f3c2d2-86bc-4305-8ddc-3d7587b8e8ac-0.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-1599b187-f60c-439b-841c-b0fa0a78da4f-0.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-0.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-1.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-0.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-1.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-0.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-1.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-2.parquet create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-1e092fe2-2c76-4682-bfec-50ebecd3957a-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-360cfb9c-6646-44cb-aec5-d156a379b4d2-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-5561631d-80ef-45bc-8b87-3d1a2876816d-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-806143e3-9271-4934-9ca5-636a815c46a0-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-a353b7ca-671b-4423-bff5-fd2888db7536-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-c09d3b21-6083-4881-a6f6-b0ecc37dac22-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-d52df85e-6f91-497a-8edd-df92bcc3e772-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-425d391e-cf9c-4770-9241-09facc794f72-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-5b55486f-5a96-408a-9079-c51b68c5dec9-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-97c08d9a-9e64-4d25-8e37-3350e3eba2aa-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-a01531dd-94c8-4b58-b844-b0e1f26a464c-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-dc356a38-74d8-41b5-a7b8-270860bf71ad-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-1 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-1 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-1 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-1 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-1 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 create mode 100644 test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-523fefd0-769f-46f8-8b58-957065ad7e20-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-523fefd0-769f-46f8-8b58-957065ad7e20-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-622c2733-874d-4ebb-b076-500a21ad0432-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-2 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-3 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-2 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-3 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-11be7772-670d-4dc1-adbb-fd8413f2779d-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-a9b54c7b-7468-4a64-97ee-32adae28681e-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8727f0fe-6af1-4c2f-8560-c38dd0a81ec0-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-bd400d22-3691-47bb-8262-753eb1a8c2c4-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-dd531b47-e547-4b27-b799-09842a4b026d-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e6ecca13-a405-472b-b1a4-77de887ae406-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-f8dc9dce-972e-41a4-b6dc-a8eae92847f4-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-2826504c-fd4a-4ef7-8f63-724e727201e9-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-1.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-6444b55f-03c4-4cdf-829a-1cd0c9d670ce-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-c469efe8-01bf-4123-a9f6-e9505701ffba-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-cda2ddb8-fcd1-4298-a4a8-4778ab7aa7a6-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-0f061bb7-3867-4575-a7f3-a649acff590b-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-42b50aa8-9cf7-409d-a011-f7d6a0cad4a8-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-d1e79d3b-bb1f-4afe-a6b3-8cc85490af74-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-043439a0-6e5d-44a8-a6b1-e43e09d661b5-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-10956d53-21db-4d51-83e6-ea39269e80ca-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-17bc0233-1c62-46c7-89f1-6093f22e9eee-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-1976c24c-7e5d-4179-9bbe-add3159334d0-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-9a39d079-dfcb-46f0-8b5b-ef1e35e78711-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-c59ddaa8-a1c3-4b09-b4a4-9f58a1accd7f-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1141d2e-69d0-4f8f-87f1-110f34c228d7-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f7ead75f-edd3-4307-9473-f5826baf5d96-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f8e3e01a-3620-40bf-9da7-e20669696822-0.parquet create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 create mode 100644 test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 25d86c0ef..4f05848b3 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -750,6 +750,7 @@ if(PAIMON_BUILD_TESTS) core/io/file_index_evaluator_test.cpp core/io/single_file_writer_test.cpp core/io/rolling_blob_file_writer_test.cpp + core/global_index/global_index_evaluator_impl_test.cpp core/global_index/indexed_split_test.cpp core/manifest/file_source_test.cpp core/manifest/file_kind_test.cpp diff --git a/src/paimon/core/global_index/global_index_evaluator_impl.cpp b/src/paimon/core/global_index/global_index_evaluator_impl.cpp index 5c1938153..c857a9f0b 100644 --- a/src/paimon/core/global_index/global_index_evaluator_impl.cpp +++ b/src/paimon/core/global_index/global_index_evaluator_impl.cpp @@ -19,21 +19,124 @@ #include "paimon/core/global_index/global_index_evaluator_impl.h" +#include +#include + #include "fmt/format.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/predicate/predicate_utils.h" namespace paimon { +namespace { +void FlattenChildren(const std::shared_ptr& compound_predicate, + std::vector>* flattened) { + for (const std::shared_ptr& child : compound_predicate->Children()) { + auto compound_child = std::dynamic_pointer_cast(child); + if (compound_child != nullptr && compound_child->GetFunction().GetType() == + compound_predicate->GetFunction().GetType()) { + FlattenChildren(compound_child, flattened); + } else { + flattened->push_back(child); + } + } +} + +/// A predicate is null-rejecting when it cannot match a row whose tested field is null. +/// Under SQL three-valued logic every comparison and match predicate rejects null; only +/// IS NULL accepts it, and IS NOT NULL is the predicate being pruned. +bool IsNullRejecting(const std::shared_ptr& predicate) { + auto leaf_predicate = std::dynamic_pointer_cast(predicate); + if (leaf_predicate == nullptr) { + return false; + } + switch (leaf_predicate->GetFunction().GetType()) { + case Function::Type::EQUAL: + case Function::Type::NOT_EQUAL: + case Function::Type::GREATER_THAN: + case Function::Type::GREATER_OR_EQUAL: + case Function::Type::LESS_THAN: + case Function::Type::LESS_OR_EQUAL: + case Function::Type::IN: + case Function::Type::NOT_IN: + case Function::Type::STARTS_WITH: + case Function::Type::ENDS_WITH: + case Function::Type::CONTAINS: + case Function::Type::LIKE: + return true; + default: + return false; + } +} + +bool IsIsNotNull(const std::shared_ptr& predicate) { + auto leaf_predicate = std::dynamic_pointer_cast(predicate); + return leaf_predicate != nullptr && + leaf_predicate->GetFunction().GetType() == Function::Type::IS_NOT_NULL; +} +} // namespace + Result> GlobalIndexEvaluatorImpl::Evaluate( const std::shared_ptr& predicate) { std::shared_ptr compound_result; if (predicate) { - PAIMON_ASSIGN_OR_RAISE(compound_result, EvaluatePredicate(predicate)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_predicate, + NormalizePredicate(predicate)); + PAIMON_ASSIGN_OR_RAISE(compound_result, EvaluatePredicate(normalized_predicate)); } return compound_result; } +Result> GlobalIndexEvaluatorImpl::NormalizePredicate( + const std::shared_ptr& predicate) { + auto compound_predicate = std::dynamic_pointer_cast(predicate); + if (compound_predicate == nullptr) { + return predicate; + } + std::vector> children; + FlattenChildren(compound_predicate, &children); + + bool is_and = compound_predicate->GetFunction().GetType() == Function::Type::AND; + if (is_and) { + std::set constrained_fields; + for (const std::shared_ptr& child : children) { + auto leaf = std::dynamic_pointer_cast(child); + if (leaf != nullptr && IsNullRejecting(child)) { + constrained_fields.insert(leaf->FieldName()); + } + } + if (!constrained_fields.empty()) { + std::vector> pruned; + pruned.reserve(children.size()); + for (const std::shared_ptr& child : children) { + auto leaf = std::dynamic_pointer_cast(child); + if (leaf != nullptr && IsIsNotNull(child) && + constrained_fields.count(leaf->FieldName()) > 0) { + continue; + } + pruned.push_back(child); + } + children = std::move(pruned); + } + } + + std::vector> normalized_children; + normalized_children.reserve(children.size()); + for (const std::shared_ptr& child : children) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_child, + NormalizePredicate(child)); + normalized_children.push_back(std::move(normalized_child)); + } + if (normalized_children.size() == 1) { + return normalized_children[0]; + } + if (is_and) { + return PredicateBuilder::And(normalized_children); + } + return PredicateBuilder::Or(normalized_children); +} + Result>> GlobalIndexEvaluatorImpl::GetIndexReaders( const std::string& field_name) { PAIMON_ASSIGN_OR_RAISE(DataField data_field, table_schema_->GetField(field_name)); diff --git a/src/paimon/core/global_index/global_index_evaluator_impl.h b/src/paimon/core/global_index/global_index_evaluator_impl.h index 7555a716e..7c1591870 100644 --- a/src/paimon/core/global_index/global_index_evaluator_impl.h +++ b/src/paimon/core/global_index/global_index_evaluator_impl.h @@ -46,6 +46,11 @@ class GlobalIndexEvaluatorImpl : public GlobalIndexEvaluator { Result> Evaluate( const std::shared_ptr& predicate) override; + /// Applies Java-compatible compound flattening and removes redundant IS NOT NULL + /// predicates from AND expressions. + static Result> NormalizePredicate( + const std::shared_ptr& predicate); + private: Result> EvaluatePredicate( const std::shared_ptr& predicate); diff --git a/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp b/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp new file mode 100644 index 000000000..46253cf23 --- /dev/null +++ b/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/global_index/global_index_evaluator_impl.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { +class RecordingGlobalIndexReader : public GlobalIndexReader { + public: + Result> VisitIsNotNull() override { + is_not_null_calls_++; + return Bitmap({0, 1}); + } + + Result> VisitIsNull() override { + is_null_calls_++; + return Bitmap({2}); + } + + Result> VisitEqual(const Literal& literal) override { + equal_calls_++; + return Bitmap({1, 3}); + } + + Result> VisitNotEqual(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitLessThan(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitLessOrEqual(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitGreaterThan(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitGreaterOrEqual( + const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitIn( + const std::vector& literals) override { + return NotEvaluable(); + } + + Result> VisitNotIn( + const std::vector& literals) override { + return NotEvaluable(); + } + + Result> VisitStartsWith(const Literal& prefix) override { + return NotEvaluable(); + } + + Result> VisitEndsWith(const Literal& suffix) override { + return NotEvaluable(); + } + + Result> VisitContains(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitLike(const Literal& literal) override { + return NotEvaluable(); + } + + Result> VisitVectorSearch( + const std::shared_ptr& vector_search) override { + return Status::Invalid("not supported"); + } + + Result> VisitFullTextSearch( + const std::shared_ptr& full_text_search) override { + return Status::Invalid("not supported"); + } + + bool IsThreadSafe() const override { + return false; + } + + std::string GetIndexType() const override { + return "test"; + } + + int32_t EqualCalls() const { + return equal_calls_; + } + + int32_t IsNullCalls() const { + return is_null_calls_; + } + + int32_t IsNotNullCalls() const { + return is_not_null_calls_; + } + + private: + static std::shared_ptr Bitmap(std::initializer_list positions) { + RoaringBitmap64 bitmap; + for (int64_t position : positions) { + bitmap.Add(position); + } + return std::make_shared( + [bitmap = std::move(bitmap)]() -> Result { return bitmap; }); + } + + static std::shared_ptr NotEvaluable() { + return nullptr; + } + + int32_t equal_calls_ = 0; + int32_t is_null_calls_ = 0; + int32_t is_not_null_calls_ = 0; +}; + +std::set CollectPositions(const std::shared_ptr& result) { + std::set positions; + EXPECT_TRUE(result != nullptr); + if (result == nullptr) { + return positions; + } + EXPECT_OK_AND_ASSIGN(std::unique_ptr iterator, + result->CreateIterator()); + while (iterator->HasNext()) { + positions.insert(iterator->Next()); + } + return positions; +} +} // namespace + +class GlobalIndexEvaluatorImplTest : public ::testing::Test { + protected: + void SetUp() override { + std::vector fields = { + DataField(0, arrow::field("a", arrow::int64())), + DataField(1, arrow::field("b", arrow::int64())), + }; + table_schema_ = std::make_shared( + /*version=*/1, /*id=*/0, fields, /*highest_field_id=*/1, + /*partition_keys=*/std::vector(), + /*primary_keys=*/std::vector(), + /*options=*/std::map(), /*comment=*/std::nullopt, + /*time_millis=*/0); + reader_ = std::make_shared(); + } + + GlobalIndexEvaluatorImpl CreateEvaluator() const { + std::shared_ptr reader = reader_; + return GlobalIndexEvaluatorImpl( + table_schema_, + [reader](int32_t field_id) -> Result>> { + if (field_id == 0) { + return std::vector>{reader}; + } + return std::vector>(); + }); + } + + std::shared_ptr Equal(const std::string& field_name, int32_t field_index) const { + return PredicateBuilder::Equal(field_index, field_name, FieldType::BIGINT, + Literal(static_cast(42))); + } + + std::shared_ptr table_schema_; + std::shared_ptr reader_; +}; + +TEST_F(GlobalIndexEvaluatorImplTest, SupportedAndUnsupportedLeavesCombineSafely) { + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + std::shared_ptr indexed = Equal("a", 0); + std::shared_ptr unindexed = Equal("b", 1); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr leaf_result, + evaluator.Evaluate(indexed)); + ASSERT_EQ((std::set{1, 3}), CollectPositions(leaf_result)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr unsupported_result, + evaluator.Evaluate(unindexed)); + ASSERT_EQ(nullptr, unsupported_result); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_predicate, + PredicateBuilder::And({indexed, unindexed})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_result, + evaluator.Evaluate(and_predicate)); + ASSERT_EQ((std::set{1, 3}), CollectPositions(and_result)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_predicate, + PredicateBuilder::Or({indexed, unindexed})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_result, + evaluator.Evaluate(or_predicate)); + ASSERT_EQ(nullptr, or_result); +} + +TEST_F(GlobalIndexEvaluatorImplTest, NormalizationFlattensNestedCompounds) { + std::shared_ptr first = Equal("a", 0); + std::shared_ptr second = PredicateBuilder::IsNull(0, "a", FieldType::BIGINT); + std::shared_ptr third = Equal("b", 1); + ASSERT_OK_AND_ASSIGN(std::shared_ptr nested, PredicateBuilder::And({first, second})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({nested, third})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr normalized, + GlobalIndexEvaluatorImpl::NormalizePredicate(predicate)); + auto compound = std::dynamic_pointer_cast(normalized); + ASSERT_TRUE(compound != nullptr); + ASSERT_EQ(Function::Type::AND, compound->GetFunction().GetType()); + ASSERT_EQ(3, compound->Children().size()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, RedundantIsNotNullIsPrunedFromAnd) { + std::shared_ptr equal = Equal("a", 0); + std::shared_ptr is_not_null = PredicateBuilder::IsNotNull(0, "a", FieldType::BIGINT); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({equal, is_not_null})); + + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator.Evaluate(predicate)); + ASSERT_EQ((std::set{1, 3}), CollectPositions(result)); + ASSERT_EQ(1, reader_->EqualCalls()); + ASSERT_EQ(0, reader_->IsNotNullCalls()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, IsNullDoesNotMakeIsNotNullRedundant) { + std::shared_ptr is_null = PredicateBuilder::IsNull(0, "a", FieldType::BIGINT); + std::shared_ptr is_not_null = PredicateBuilder::IsNotNull(0, "a", FieldType::BIGINT); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({is_null, is_not_null})); + + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator.Evaluate(predicate)); + ASSERT_TRUE(CollectPositions(result).empty()); + ASSERT_EQ(1, reader_->IsNullCalls()); + ASSERT_EQ(1, reader_->IsNotNullCalls()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, OrDoesNotPruneIsNotNull) { + std::shared_ptr equal = Equal("a", 0); + std::shared_ptr is_not_null = PredicateBuilder::IsNotNull(0, "a", FieldType::BIGINT); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::Or({equal, is_not_null})); + + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, evaluator.Evaluate(predicate)); + ASSERT_EQ((std::set{0, 1, 3}), CollectPositions(result)); + ASSERT_EQ(1, reader_->EqualCalls()); + ASSERT_EQ(1, reader_->IsNotNullCalls()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, NullPredicateIsNotEvaluated) { + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + evaluator.Evaluate(/*predicate=*/nullptr)); + ASSERT_EQ(nullptr, result); +} + +} // namespace paimon::test diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.cpp b/src/paimon/core/index/pk/primary_key_index_definitions.cpp index 8f87184ba..214338f4b 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions.cpp +++ b/src/paimon/core/index/pk/primary_key_index_definitions.cpp @@ -210,8 +210,13 @@ Result PrimaryKeyIndexDefinitions::Create(const Tabl } std::vector PrimaryKeyIndexDefinitions::ScalarDefinitions() const { + return ScalarDefinitions(definitions_); +} + +std::vector PrimaryKeyIndexDefinitions::ScalarDefinitions( + const std::vector& definitions) { std::vector scalar_definitions; - for (const PrimaryKeyIndexDefinition& definition : definitions_) { + for (const PrimaryKeyIndexDefinition& definition : definitions) { if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE || definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BITMAP) { scalar_definitions.push_back(definition); diff --git a/src/paimon/core/index/pk/primary_key_index_definitions.h b/src/paimon/core/index/pk/primary_key_index_definitions.h index 37f20da5f..247c6b5f4 100644 --- a/src/paimon/core/index/pk/primary_key_index_definitions.h +++ b/src/paimon/core/index/pk/primary_key_index_definitions.h @@ -45,6 +45,10 @@ class PrimaryKeyIndexDefinitions { /// @return The scalar (BTree / Bitmap) definitions usable by batch scans. std::vector ScalarDefinitions() const; + /// Filters an arbitrary definition list to the scalar families usable by batch scans. + static std::vector ScalarDefinitions( + const std::vector& definitions); + private: explicit PrimaryKeyIndexDefinitions(std::vector definitions) : definitions_(std::move(definitions)) {} diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp index 1751eef7f..7d3b5a6d1 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.cpp @@ -80,18 +80,18 @@ PkSortedBucketIndexState PkSortedBucketIndexState::FromActiveDataFiles( payload_metas_by_level[source_meta.DataLevel()].push_back(std::move(source_meta)); } - std::vector groups; + std::vector> groups; std::set covered_levels; for (const auto& level_payloads : payloads_by_level) { int32_t level = level_payloads.first; - std::optional group; + std::shared_ptr group; if (level_payloads.second.size() == 1) { group = PkSortedIndexGroup::Create(field_id, index_type, sources_by_level[level], level_payloads.second[0], payload_metas_by_level[level][0]); } - if (group != std::nullopt) { - groups.push_back(std::move(group).value()); + if (group != nullptr) { + groups.push_back(std::move(group)); covered_levels.insert(level); } else { rejected.insert(rejected.end(), level_payloads.second.begin(), diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h index 42250288d..6923010da 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state.h @@ -44,7 +44,7 @@ class PkSortedBucketIndexState { const std::vector>& active_data_files, const std::vector>& active_payloads); - const std::vector& Groups() const { + const std::vector>& Groups() const { return groups_; } @@ -61,7 +61,7 @@ class PkSortedBucketIndexState { } private: - PkSortedBucketIndexState(std::vector groups, + PkSortedBucketIndexState(std::vector> groups, std::vector covered_source_files, std::vector uncovered_source_files, std::vector> rejected_payloads) @@ -70,7 +70,7 @@ class PkSortedBucketIndexState { uncovered_source_files_(std::move(uncovered_source_files)), rejected_payloads_(std::move(rejected_payloads)) {} - std::vector groups_; + std::vector> groups_; std::vector covered_source_files_; std::vector uncovered_source_files_; std::vector> rejected_payloads_; diff --git a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp index d85045cac..d01f18c44 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_bucket_index_state_test.cpp @@ -105,11 +105,11 @@ TEST_F(PkSortedBucketIndexStateTest, BuildsGroupWhenPayloadMatchesLevelSources) PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles(7, "btree", data_files, {payload}); ASSERT_EQ(1, state.Groups().size()); - const PkSortedIndexGroup& group = state.Groups()[0]; - ASSERT_EQ(5, group.DataLevel()); - ASSERT_EQ(300, group.TotalSourceRowCount()); - ASSERT_EQ(expected_sources, group.SourceFiles()); - ASSERT_EQ(payload, group.Payload()); + const std::shared_ptr& group = state.Groups()[0]; + ASSERT_EQ(5, group->DataLevel()); + ASSERT_EQ(300, group->TotalSourceRowCount()); + ASSERT_EQ(expected_sources, group->SourceFiles()); + ASSERT_EQ(payload, group->Payload()); ASSERT_EQ(expected_sources, state.CoveredSourceFiles()); ASSERT_TRUE(state.UncoveredSourceFiles().empty()); ASSERT_TRUE(state.RejectedPayloads().empty()); @@ -222,7 +222,7 @@ TEST_F(PkSortedBucketIndexStateTest, WrongCandidateDoesNotMaskValidPayload) { PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( 7, "btree", data_files, {valid_payload, wrong_payload}); ASSERT_EQ(1, state.Groups().size()); - ASSERT_EQ(valid_payload, state.Groups()[0].Payload()); + ASSERT_EQ(valid_payload, state.Groups()[0]->Payload()); ASSERT_EQ(1, state.RejectedPayloads().size()); ASSERT_EQ(wrong_payload, state.RejectedPayloads()[0]); ASSERT_TRUE(state.UncoveredSourceFiles().empty()); @@ -290,8 +290,8 @@ TEST_F(PkSortedBucketIndexStateTest, KeepsValidLevelAndLeavesBrokenLevelUncovere PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( 7, "btree", data_files, {valid_payload, broken_payload}); ASSERT_EQ(1, state.Groups().size()); - ASSERT_EQ(4, state.Groups()[0].DataLevel()); - ASSERT_EQ(valid_payload, state.Groups()[0].Payload()); + ASSERT_EQ(4, state.Groups()[0]->DataLevel()); + ASSERT_EQ(valid_payload, state.Groups()[0]->Payload()); std::vector expected_covered = {{"c", 50}}; ASSERT_EQ(expected_covered, state.CoveredSourceFiles()); std::vector expected_uncovered = {{"a", 100}, {"b", 200}}; diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp index f919995b1..7d3b7aada 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_index_file.cpp @@ -43,6 +43,8 @@ Result> PkSortedIndexFile::Build( const std::shared_ptr& sorted_values, std::vector sorted_ordinals, const std::shared_ptr& file_writer, bool is_external_path, const std::shared_ptr& pool) { + // TODO(wangyong9999): Replace the all-in-memory sorted values and ordinals with an + // external sort buffer and feed the index writer in bounded batches. PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexSourceMeta source_meta, PrimaryKeyIndexSourceMeta::Create(data_level, source_files)); int64_t source_row_count = 0; @@ -103,7 +105,8 @@ Result> PkSortedIndexFile::Build( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr source_meta_bytes, source_meta.Serialize(pool)); std::optional external_path; if (is_external_path) { - external_path = io_meta.file_path; + PAIMON_ASSIGN_OR_RAISE(Path path, PathUtil::ToPath(io_meta.file_path)); + external_path = path.ToString(); } return std::make_shared( index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, source_row_count, diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp b/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp index b226dc11d..851457bef 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp +++ b/src/paimon/core/index/pksorted/pk_sorted_index_group.cpp @@ -23,22 +23,22 @@ #include namespace paimon { -std::optional PkSortedIndexGroup::Create( +std::shared_ptr PkSortedIndexGroup::Create( int32_t field_id, const std::string& index_type, const std::vector& expected_sources, const std::shared_ptr& payload, const PrimaryKeyIndexSourceMeta& payload_source_meta) { if (payload == nullptr || expected_sources.empty()) { - return std::nullopt; + return nullptr; } int64_t source_row_count = 0; std::set source_names; for (const PrimaryKeyIndexSourceFile& source_file : expected_sources) { if (!source_names.insert(source_file.file_name).second) { - return std::nullopt; + return nullptr; } if (__builtin_add_overflow(source_row_count, source_file.row_count, &source_row_count)) { - return std::nullopt; + return nullptr; } } @@ -48,10 +48,10 @@ std::optional PkSortedIndexGroup::Create( meta.value().index_field_id != field_id || meta.value().row_range_start != 0 || meta.value().row_range_end != source_row_count - 1 || payload->RowCount() != source_row_count) { - return std::nullopt; + return nullptr; } - return PkSortedIndexGroup(payload_source_meta.DataLevel(), expected_sources, payload, - source_row_count); + return std::shared_ptr(new PkSortedIndexGroup( + payload_source_meta.DataLevel(), expected_sources, payload, source_row_count)); } } // namespace paimon diff --git a/src/paimon/core/index/pksorted/pk_sorted_index_group.h b/src/paimon/core/index/pksorted/pk_sorted_index_group.h index e202f8956..9680735dc 100644 --- a/src/paimon/core/index/pksorted/pk_sorted_index_group.h +++ b/src/paimon/core/index/pksorted/pk_sorted_index_group.h @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -40,9 +39,9 @@ namespace paimon { /// source row count sum. Anything else must be treated as uncovered. class PkSortedIndexGroup { public: - /// Validates one payload against the expected level sources; returns `std::nullopt` - /// when any coverage condition fails. - static std::optional Create( + /// Validates one payload against the expected level sources; returns null when any + /// coverage condition fails. + static std::shared_ptr Create( int32_t field_id, const std::string& index_type, const std::vector& expected_sources, const std::shared_ptr& payload, diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index c0cd6a3c0..82951d6e6 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "arrow/c/bridge.h" @@ -297,7 +298,8 @@ Result> AppendOnlyFileStoreWrite::CreateFilesReader auto read = std::make_unique(file_store_path_factory_, internal_read_context, pool_, compact_executor_); - return read->CreateReader(partition, bucket, files, dv_factory); + return read->CreateReader(partition, bucket, files, dv_factory, + /*local_row_ranges=*/std::nullopt); } } // namespace paimon diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index 8bb59987e..a1effa8c4 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -55,6 +55,30 @@ class DataFilePathFactory; class Executor; class Predicate; +namespace { + +Status ValidateFileLocalRowRanges(const std::vector>& data_files, + const std::optional>& local_row_ranges) { + if (local_row_ranges == std::nullopt) { + return Status::OK(); + } + if (data_files.size() != 1) { + return Status::Invalid("file-local row ranges require exactly one data file"); + } + const auto& file = data_files.front(); + for (const Range& range : local_row_ranges.value()) { + if (range.from < 0 || range.to < range.from || range.to >= file->row_count || + range.to >= std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("Invalid file-local row range [{}, {}] for file {} with {} rows.", + range.from, range.to, file->file_name, file->row_count)); + } + } + return Status::OK(); +} + +} // namespace + RawFileSplitRead::RawFileSplitRead(const std::shared_ptr& path_factory, const std::shared_ptr& context, const std::shared_ptr& memory_pool, @@ -69,6 +93,12 @@ Result> RawFileSplitRead::CreateReader( const std::shared_ptr& split) { if (auto indexed_split = std::dynamic_pointer_cast(split)) { PAIMON_RETURN_NOT_OK(indexed_split->Validate()); + if (!indexed_split->Scores().empty()) { + // TODO(wangyong9999): Propagate indexed scores through the primary-key + // physical-position read path. + return Status::NotImplemented( + "Primary-key reads do not support scored indexed splits yet."); + } const std::shared_ptr& inner_split = indexed_split->GetDataSplit(); auto inner_split_impl = std::dynamic_pointer_cast(inner_split); if (!inner_split_impl) { @@ -87,13 +117,14 @@ Result> RawFileSplitRead::CreateReader( return Status::Invalid("cannot cast split to data_split in RawFileSplitRead"); } return CreateReader(data_split->Partition(), data_split->Bucket(), data_split->DataFiles(), - data_split->DeletionFiles()); + data_split->DeletionFiles(), /*local_row_ranges=*/std::nullopt); } Result> RawFileSplitRead::CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& data_files, DeletionVector::Factory dv_factory, const std::optional>& local_row_ranges) { + PAIMON_RETURN_NOT_OK(ValidateFileLocalRowRanges(data_files, local_row_ranges)); const auto& predicate = context_->GetPredicate(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, path_factory_->CreateDataFilePathFactory(partition, bucket)); @@ -112,14 +143,6 @@ Result> RawFileSplitRead::CreateReader( return std::make_unique(std::move(batch_reader), pool_); } -Result> RawFileSplitRead::CreateReader( - const BinaryRow& partition, int32_t bucket, - const std::vector>& data_files, - const std::vector>& deletion_files) { - return CreateReader(partition, bucket, data_files, deletion_files, - /*local_row_ranges=*/std::nullopt); -} - Result> RawFileSplitRead::CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& data_files, @@ -184,12 +207,6 @@ Result> RawFileSplitRead::ApplyIndexAndDvReader if (ranges != std::nullopt) { RoaringBitmap32 ranges_bitmap; for (const Range& range : ranges.value()) { - if (range.from < 0 || range.to < range.from || - range.to >= std::numeric_limits::max()) { - return Status::Invalid( - fmt::format("Invalid file-local row range [{}, {}] for file {}.", range.from, - range.to, file->file_name)); - } ranges_bitmap.AddRange(static_cast(range.from), static_cast(range.to + 1)); } diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 9b87dc835..ac211b257 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -67,13 +67,9 @@ class RawFileSplitRead : public AbstractSplitRead { /// Also accepts an `IndexedSplit` over a single-file data split, in which case its row /// ranges narrow the read to the given file-local physical positions. Result> CreateReader(const std::shared_ptr& split) override; - Result> CreateReader( - const BinaryRow& partition, int32_t bucket, - const std::vector>& files, - const std::vector>& deletion_files); - /// Reads with an optional selection of file-local row positions. The ranges apply to - /// every file of the split, so callers pass them only for single-file splits. + /// Reads with an optional selection of file-local row positions. A range selection + /// requires exactly one data file. Result> CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& files, @@ -83,7 +79,7 @@ class RawFileSplitRead : public AbstractSplitRead { Result> CreateReader( const BinaryRow& partition, int32_t bucket, const std::vector>& files, DeletionVector::Factory dv_factory, - const std::optional>& local_row_ranges = std::nullopt); + const std::optional>& local_row_ranges); Result Match(const std::shared_ptr& split, bool force_keep_delete) const override; diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 569f8dfd4..29dc0f561 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -30,6 +30,7 @@ #include "paimon/common/reader/concat_batch_reader.h" #include "paimon/common/types/data_field.h" #include "paimon/core/core_options.h" +#include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/operation/internal_read_context.h" @@ -505,6 +506,22 @@ TEST_F(RawFileSplitReadTest, TestMatch) { split_read->Match(data_split, /*force_keep_delete=*/false)); ASSERT_FALSE(match_result); } + { + auto data_split = std::dynamic_pointer_cast( + create_data_split(/*is_streaming=*/false, /*raw_convertible=*/false)); + auto indexed_split = std::make_shared( + data_split, std::vector{Range(0, 0)}, std::vector{0.5F}); + ASSERT_NOK_WITH_MSG(split_read->CreateReader(indexed_split), + "Primary-key reads do not support scored indexed splits yet"); + } + { + auto data_split = std::dynamic_pointer_cast( + create_data_split(/*is_streaming=*/false, /*raw_convertible=*/false)); + auto indexed_split = + std::make_shared(data_split, std::vector{Range(0, 1)}); + ASSERT_NOK_WITH_MSG(split_read->CreateReader(indexed_split), + "Invalid file-local row range [0, 1]"); + } { ASSERT_NOK(split_read->Match(nullptr, /*force_keep_delete=*/false)); } diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp index ad085fb3f..f3a1b7569 100644 --- a/src/paimon/core/table/source/key_value_table_read.cpp +++ b/src/paimon/core/table/source/key_value_table_read.cpp @@ -77,21 +77,29 @@ Result> KeyValueTableRead::CreateReader( const std::shared_ptr& split) { std::shared_ptr dispatch_split = split; if (auto indexed_split = std::dynamic_pointer_cast(split)) { - // A primary-key sorted-index split narrows one raw-readable file to file-local row - // positions. If the raw read cannot serve the inner split, fall back to reading the - // whole file: the index only narrows the scan, so the unnarrowed read stays correct. + PAIMON_RETURN_NOT_OK(indexed_split->Validate()); + if (!indexed_split->Scores().empty()) { + // TODO(wangyong9999): Propagate indexed scores through the primary-key + // physical-position read path. + return Status::NotImplemented( + "Primary-key reads do not support scored indexed splits yet."); + } + // Primary-key indexed splits carry physical positions and are routed independently + // of the inner split's raw-convertible marker, matching Java's dedicated provider. const std::shared_ptr& inner_split = indexed_split->GetDataSplit(); - for (const auto& read : split_reads_) { - auto* raw_read = dynamic_cast(read.get()); - if (raw_read == nullptr) { - continue; - } - PAIMON_ASSIGN_OR_RAISE(bool matched, read->Match(inner_split, force_keep_delete_)); - if (matched) { - return read->CreateReader(indexed_split); + if (!force_keep_delete_) { + for (const auto& read : split_reads_) { + if (dynamic_cast(read.get()) != nullptr) { + return read->CreateReader(indexed_split); + } } + return Status::Invalid( + "create reader failed, primary-key indexed split has no raw reader."); + } else { + // Keeping delete rows is incompatible with physical-position pruning. Reading the + // inner split through the normal merge path preserves correctness. + dispatch_split = inner_split; } - dispatch_split = inner_split; } auto data_split = std::dynamic_pointer_cast(dispatch_split); if (!data_split) { diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp index 9425c1a06..e65b6aa30 100644 --- a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp @@ -36,9 +36,7 @@ #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/executor.h" -#include "paimon/predicate/compound_predicate.h" -#include "paimon/predicate/leaf_predicate.h" -#include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/predicate_utils.h" namespace paimon { namespace { @@ -58,148 +56,6 @@ Result> CreateGlobalIndexExecutor(const CoreOptions& c return executor; } -/// Restricts a predicate to leaves over the indexed fields: an AND keeps its convertible -/// children, an OR is only kept when every child is convertible, and everything else is -/// dropped. A null return means no part of the predicate can use the index. -Result> ProjectToIndexedFields( - const std::shared_ptr& predicate, const std::set& indexed_fields) { - if (predicate == nullptr) { - return std::shared_ptr(nullptr); - } - if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { - if (indexed_fields.count(leaf_predicate->FieldName()) > 0) { - return predicate; - } - return std::shared_ptr(nullptr); - } - auto compound_predicate = std::dynamic_pointer_cast(predicate); - if (compound_predicate == nullptr) { - return std::shared_ptr(nullptr); - } - bool is_and = compound_predicate->GetFunction().GetType() == Function::Type::AND; - bool is_or = compound_predicate->GetFunction().GetType() == Function::Type::OR; - if (!is_and && !is_or) { - return std::shared_ptr(nullptr); - } - std::vector> converted_children; - for (const std::shared_ptr& child : compound_predicate->Children()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converted_child, - ProjectToIndexedFields(child, indexed_fields)); - if (converted_child != nullptr) { - converted_children.push_back(std::move(converted_child)); - } else if (is_or) { - return std::shared_ptr(nullptr); - } - } - if (converted_children.empty()) { - return std::shared_ptr(nullptr); - } - if (converted_children.size() == 1) { - return converted_children[0]; - } - if (is_and) { - return PredicateBuilder::And(converted_children); - } - return PredicateBuilder::Or(converted_children); -} - -void FlattenChildren(const std::shared_ptr& compound_predicate, - std::vector>* flattened) { - for (const std::shared_ptr& child : compound_predicate->Children()) { - auto compound_child = std::dynamic_pointer_cast(child); - if (compound_child != nullptr && compound_child->GetFunction().GetType() == - compound_predicate->GetFunction().GetType()) { - FlattenChildren(compound_child, flattened); - } else { - flattened->push_back(child); - } - } -} - -/// A predicate is null-rejecting when it cannot match a row whose tested field is null. -/// Under SQL three-valued logic every comparison and match predicate rejects null; only -/// IS NULL accepts it, and IS NOT NULL is the predicate being pruned. -bool IsNullRejecting(const std::shared_ptr& predicate) { - auto leaf_predicate = std::dynamic_pointer_cast(predicate); - if (leaf_predicate == nullptr) { - return false; - } - switch (leaf_predicate->GetFunction().GetType()) { - case Function::Type::EQUAL: - case Function::Type::NOT_EQUAL: - case Function::Type::GREATER_THAN: - case Function::Type::GREATER_OR_EQUAL: - case Function::Type::LESS_THAN: - case Function::Type::LESS_OR_EQUAL: - case Function::Type::IN: - case Function::Type::NOT_IN: - case Function::Type::STARTS_WITH: - case Function::Type::ENDS_WITH: - case Function::Type::CONTAINS: - case Function::Type::LIKE: - return true; - default: - return false; - } -} - -bool IsIsNotNull(const std::shared_ptr& predicate) { - auto leaf_predicate = std::dynamic_pointer_cast(predicate); - return leaf_predicate != nullptr && - leaf_predicate->GetFunction().GetType() == Function::Type::IS_NOT_NULL; -} - -/// Flattens nested same-function compounds and, inside an AND, removes `f IS NOT NULL` -/// leaves made redundant by a null-rejecting sibling on the same field. Pruning must not -/// consider `f IS NULL` as constraining: dropping IS NOT NULL from -/// "f IS NULL AND f IS NOT NULL" would turn the empty result into the set of null rows. -Result> NormalizePredicate(const std::shared_ptr& predicate) { - auto compound_predicate = std::dynamic_pointer_cast(predicate); - if (compound_predicate == nullptr) { - return predicate; - } - std::vector> children; - FlattenChildren(compound_predicate, &children); - - bool is_and = compound_predicate->GetFunction().GetType() == Function::Type::AND; - if (is_and) { - std::set constrained_fields; - for (const std::shared_ptr& child : children) { - if (IsNullRejecting(child)) { - constrained_fields.insert( - std::dynamic_pointer_cast(child)->FieldName()); - } - } - if (!constrained_fields.empty()) { - std::vector> pruned; - pruned.reserve(children.size()); - for (const std::shared_ptr& child : children) { - if (IsIsNotNull(child) && - constrained_fields.count( - std::dynamic_pointer_cast(child)->FieldName()) > 0) { - continue; - } - pruned.push_back(child); - } - children = std::move(pruned); - } - } - - std::vector> normalized_children; - normalized_children.reserve(children.size()); - for (const std::shared_ptr& child : children) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_child, - NormalizePredicate(child)); - normalized_children.push_back(std::move(normalized_child)); - } - if (normalized_children.size() == 1) { - return normalized_children[0]; - } - if (is_and) { - return PredicateBuilder::And(normalized_children); - } - return PredicateBuilder::Or(normalized_children); -} } // namespace Result> PrimaryKeyIndexBatchScan::Create( @@ -228,13 +84,15 @@ Result> PrimaryKeyIndexBatchScan::CreatePlan() { indexed_fields.insert(definition.Column()); indexed_field_ids.insert(definition.FieldId()); } - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr index_predicate, - ProjectToIndexedFields(batch_scan_->GetNonPartitionPredicate(), indexed_fields)); - if (index_predicate == nullptr) { + const std::shared_ptr& predicate = batch_scan_->GetNonPartitionPredicate(); + if (predicate == nullptr) { + return data_plan; + } + Result contains_indexed_field = + PredicateUtils::ContainAnyField(predicate, indexed_fields); + if (!contains_indexed_field.ok() || !contains_indexed_field.value()) { return data_plan; } - PAIMON_ASSIGN_OR_RAISE(index_predicate, NormalizePredicate(index_predicate)); std::vector> data_splits; data_splits.reserve(data_plan->Splits().size()); @@ -288,7 +146,7 @@ Result> PrimaryKeyIndexBatchScan::CreatePlan() { table_schema_, pool_, executor); PAIMON_ASSIGN_OR_RAISE( PrimaryKeySortedIndexScan::EvaluatedPlan evaluated_plan, - PrimaryKeySortedIndexScan::Evaluate(index_plan, table_schema_, index_predicate, + PrimaryKeySortedIndexScan::Evaluate(index_plan, table_schema_, predicate, scalar_definitions_, reader_factory)); PAIMON_ASSIGN_OR_RAISE(std::vector> splits, PrimaryKeySortedIndexResult::ToSplits(evaluated_plan)); diff --git a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp index 2814b3325..557cd4d4e 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_result.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_result.cpp @@ -43,7 +43,7 @@ Result> ToSingleFileSplit( builder.WithSnapshot(source->SnapshotId()) .WithTotalBuckets(source->TotalBuckets()) .IsStreaming(false) - .RawConvertible(source->RawConvertible()); + .RawConvertible(false); if (!source->DeletionFiles().empty()) { builder.WithDataDeletionFiles({source->DeletionFiles()[file.FileIndex()]}); } diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp index 8b56625d4..e2af0a2c8 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -32,6 +32,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/global_index/global_index_evaluator_impl.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" #include "paimon/core/index/pksorted/pk_sorted_bucket_index_state.h" #include "paimon/core/manifest/file_kind.h" #include "paimon/global_index/bitmap_global_index_result.h" @@ -45,25 +46,8 @@ namespace paimon { namespace { using BucketKey = std::pair; -enum class QueryOperation { - IS_NOT_NULL, - IS_NULL, - EQUAL, - NOT_EQUAL, - LESS_THAN, - LESS_OR_EQUAL, - GREATER_THAN, - GREATER_OR_EQUAL, - IN, - NOT_IN, - STARTS_WITH, - ENDS_WITH, - CONTAINS, - LIKE, -}; - struct QueryKey { - QueryOperation operation; + Function::Type operation; std::vector literals; bool operator==(const QueryKey& other) const { @@ -211,84 +195,84 @@ class FileLocalGroupReader : public GlobalIndexReader { : shared_reader_(std::move(shared_reader)), source_index_(source_index) {} Result> VisitIsNotNull() override { - return Query({QueryOperation::IS_NOT_NULL, {}}, + return Query({Function::Type::IS_NOT_NULL, {}}, [](GlobalIndexReader* reader) { return reader->VisitIsNotNull(); }); } Result> VisitIsNull() override { - return Query({QueryOperation::IS_NULL, {}}, + return Query({Function::Type::IS_NULL, {}}, [](GlobalIndexReader* reader) { return reader->VisitIsNull(); }); } Result> VisitEqual(const Literal& literal) override { - return Query({QueryOperation::EQUAL, {literal}}, + return Query({Function::Type::EQUAL, {literal}}, [&literal](GlobalIndexReader* reader) { return reader->VisitEqual(literal); }); } Result> VisitNotEqual(const Literal& literal) override { - return Query({QueryOperation::NOT_EQUAL, {literal}}, [&literal](GlobalIndexReader* reader) { + return Query({Function::Type::NOT_EQUAL, {literal}}, [&literal](GlobalIndexReader* reader) { return reader->VisitNotEqual(literal); }); } Result> VisitLessThan(const Literal& literal) override { - return Query({QueryOperation::LESS_THAN, {literal}}, [&literal](GlobalIndexReader* reader) { + return Query({Function::Type::LESS_THAN, {literal}}, [&literal](GlobalIndexReader* reader) { return reader->VisitLessThan(literal); }); } Result> VisitLessOrEqual(const Literal& literal) override { return Query( - {QueryOperation::LESS_OR_EQUAL, {literal}}, + {Function::Type::LESS_OR_EQUAL, {literal}}, [&literal](GlobalIndexReader* reader) { return reader->VisitLessOrEqual(literal); }); } Result> VisitGreaterThan(const Literal& literal) override { return Query( - {QueryOperation::GREATER_THAN, {literal}}, + {Function::Type::GREATER_THAN, {literal}}, [&literal](GlobalIndexReader* reader) { return reader->VisitGreaterThan(literal); }); } Result> VisitGreaterOrEqual( const Literal& literal) override { return Query( - {QueryOperation::GREATER_OR_EQUAL, {literal}}, + {Function::Type::GREATER_OR_EQUAL, {literal}}, [&literal](GlobalIndexReader* reader) { return reader->VisitGreaterOrEqual(literal); }); } Result> VisitIn( const std::vector& literals) override { - return Query({QueryOperation::IN, literals}, + return Query({Function::Type::IN, literals}, [&literals](GlobalIndexReader* reader) { return reader->VisitIn(literals); }); } Result> VisitNotIn( const std::vector& literals) override { - return Query({QueryOperation::NOT_IN, literals}, [&literals](GlobalIndexReader* reader) { + return Query({Function::Type::NOT_IN, literals}, [&literals](GlobalIndexReader* reader) { return reader->VisitNotIn(literals); }); } Result> VisitStartsWith(const Literal& prefix) override { - return Query({QueryOperation::STARTS_WITH, {prefix}}, [&prefix](GlobalIndexReader* reader) { + return Query({Function::Type::STARTS_WITH, {prefix}}, [&prefix](GlobalIndexReader* reader) { return reader->VisitStartsWith(prefix); }); } Result> VisitEndsWith(const Literal& suffix) override { - return Query({QueryOperation::ENDS_WITH, {suffix}}, [&suffix](GlobalIndexReader* reader) { + return Query({Function::Type::ENDS_WITH, {suffix}}, [&suffix](GlobalIndexReader* reader) { return reader->VisitEndsWith(suffix); }); } Result> VisitContains(const Literal& literal) override { - return Query({QueryOperation::CONTAINS, {literal}}, [&literal](GlobalIndexReader* reader) { + return Query({Function::Type::CONTAINS, {literal}}, [&literal](GlobalIndexReader* reader) { return reader->VisitContains(literal); }); } Result> VisitLike(const Literal& literal) override { - return Query({QueryOperation::LIKE, {literal}}, + return Query({Function::Type::LIKE, {literal}}, [&literal](GlobalIndexReader* reader) { return reader->VisitLike(literal); }); } @@ -354,13 +338,8 @@ Result PrimaryKeySortedIndexScan::CreatePlan( payloads_by_bucket[BucketKey(entry.partition, entry.bucket)].push_back(payload); } - std::vector scalar_definitions; - for (const PrimaryKeyIndexDefinition& definition : definitions) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE || - definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BITMAP) { - scalar_definitions.push_back(definition); - } - } + std::vector scalar_definitions = + PrimaryKeyIndexDefinitions::ScalarDefinitions(definitions); std::unordered_map>> data_files_by_bucket; for (const std::shared_ptr& split : data_splits) { @@ -414,14 +393,13 @@ Result PrimaryKeySortedIndexScan::CreatePlan( PkSortedBucketIndexState state = PkSortedBucketIndexState::FromActiveDataFiles( definition.FieldId(), definition.IndexType(), bucket_entry.second, definition_payloads); - for (const PkSortedIndexGroup& group : state.Groups()) { - auto shared_group = std::make_shared(group); - for (const PrimaryKeyIndexSourceFile& source_file : group.SourceFiles()) { + for (const std::shared_ptr& group : state.Groups()) { + for (const PrimaryKeyIndexSourceFile& source_file : group->SourceFiles()) { if (active_source_files.count({source_file.file_name, source_file.row_count}) == 0) { continue; } - groups_by_source[source_file.file_name][definition.FieldId()] = shared_group; + groups_by_source[source_file.file_name][definition.FieldId()] = group; } } } @@ -452,11 +430,9 @@ Result PrimaryKeySortedIndexScan::Eval const std::vector& definitions, const ReaderFactory& reader_factory) { std::map definitions_by_field; - for (const PrimaryKeyIndexDefinition& definition : definitions) { - if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BTREE || - definition.GetFamily() == PrimaryKeyIndexDefinition::Family::BITMAP) { - definitions_by_field.emplace(definition.FieldId(), definition); - } + for (const PrimaryKeyIndexDefinition& definition : + PrimaryKeyIndexDefinitions::ScalarDefinitions(definitions)) { + definitions_by_field.emplace(definition.FieldId(), definition); } std::unordered_map> diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index fdb6ffc7e..5ef555bfe 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/table/source/primary_key_sorted_index_scan.h" +#include #include #include #include @@ -27,6 +28,7 @@ #include "arrow/api.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" #include "paimon/core/index/pksorted/pk_sorted_index_file.h" @@ -100,6 +102,9 @@ class StubGlobalIndexReader : public GlobalIndexReader { explicit StubGlobalIndexReader(RoaringBitmap64 equal_result) : equal_result_(std::move(equal_result)) {} + StubGlobalIndexReader(RoaringBitmap64 equal_result, std::shared_ptr equal_call_count) + : equal_result_(std::move(equal_result)), equal_call_count_(std::move(equal_call_count)) {} + Result> VisitIsNotNull() override { return NotEvaluable(); } @@ -107,6 +112,9 @@ class StubGlobalIndexReader : public GlobalIndexReader { return NotEvaluable(); } Result> VisitEqual(const Literal& literal) override { + if (equal_call_count_ != nullptr) { + (*equal_call_count_)++; + } RoaringBitmap64 copy = equal_result_; return std::make_shared( [bitmap = std::move(copy)]() -> Result { return bitmap; }); @@ -168,6 +176,7 @@ class StubGlobalIndexReader : public GlobalIndexReader { } RoaringBitmap64 equal_result_; + std::shared_ptr equal_call_count_; }; } // namespace @@ -210,7 +219,9 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); } - Result> BuildPayload(std::vector ordinals) { + Result> BuildPayload(std::vector ordinals, + const std::string& writer_base_path, + bool is_external_path) { std::vector source_files = {{"a.parquet", kFileARows}, {"b.parquet", kFileBRows}}; arrow::Int64Builder values_builder; @@ -220,11 +231,14 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { std::shared_ptr sorted_values; PAIMON_RETURN_NOT_OK_FROM_ARROW(values_builder.Finish(&sorted_values)); PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema_->GetField(kPriceFieldId)); - auto file_writer = std::make_shared(fs_, base_path_); + auto file_writer = std::make_shared(fs_, writer_base_path); return PkSortedIndexFile::Build(field, "btree", definitions_[0].Options(), /*data_level=*/5, source_files, sorted_values, - std::move(ordinals), file_writer, - /*is_external_path=*/false, pool_); + std::move(ordinals), file_writer, is_external_path, pool_); + } + + Result> BuildPayload(std::vector ordinals) { + return BuildPayload(std::move(ordinals), base_path_, /*is_external_path=*/false); } /// Builds the standard payload of this fixture: sources a.parquet(100) + b.parquet(200) @@ -326,6 +340,7 @@ TEST_F(PrimaryKeySortedIndexScanTest, EqualNarrowsToSingleFileRange) { ASSERT_TRUE(indexed_split != nullptr); auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); ASSERT_TRUE(inner_split != nullptr); + ASSERT_FALSE(inner_split->RawConvertible()); ASSERT_EQ(inner_split->DataFiles().size(), 1); ASSERT_EQ(inner_split->DataFiles()[0]->file_name, "a.parquet"); ASSERT_EQ(indexed_split->RowRanges().size(), 1); @@ -340,8 +355,23 @@ TEST_F(PrimaryKeySortedIndexScanTest, BuildRejectsDuplicateOrdinals) { ordinals.push_back(i); } ordinals[1] = 0; - ASSERT_NOK_WITH_MSG(BuildPayload(std::move(ordinals)).status(), - "Row id 0 appears more than once"); + ASSERT_NOK_WITH_MSG(BuildPayload(std::move(ordinals)), "Row id 0 appears more than once"); +} + +TEST_F(PrimaryKeySortedIndexScanTest, ExternalPayloadPathIsNormalized) { + std::vector ordinals; + ordinals.reserve(kTotalRows); + for (int64_t i = 0; i < kTotalRows; i++) { + ordinals.push_back(i); + } + std::string writer_base_path = base_path_ + "//"; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr payload, + BuildPayload(std::move(ordinals), writer_base_path, /*is_external_path=*/true)); + ASSERT_TRUE(payload->ExternalPath().has_value()); + ASSERT_OK_AND_ASSIGN(std::string normalized_path, + PathUtil::NormalizePath(writer_base_path + "/" + payload->FileName())); + ASSERT_EQ(normalized_path, payload->ExternalPath().value()); } TEST_F(PrimaryKeySortedIndexScanTest, RangeSpansFileBoundary) { @@ -374,6 +404,33 @@ TEST_F(PrimaryKeySortedIndexScanTest, RangeSpansFileBoundary) { ASSERT_EQ(indexed_b->RowRanges()[0].to, 5); } +TEST_F(PrimaryKeySortedIndexScanTest, GroupAndQueryAreSharedAcrossSourceFiles) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN(PrimaryKeySortedIndexScan::Plan plan, + PrimaryKeySortedIndexScan::CreatePlan(kSnapshotId, {split}, definitions_, + MakeEntries(payload))); + ASSERT_EQ(2, plan.Files().size()); + ASSERT_EQ(plan.Files()[0].Group(kPriceFieldId), plan.Files()[1].Group(kPriceFieldId)); + + RoaringBitmap64 positions; + positions.Add(5); + auto equal_call_count = std::make_shared(0); + PrimaryKeySortedIndexScan::ReaderFactory reader_factory = + [positions, equal_call_count]( + const PrimaryKeySortedIndexScan::FilePlan& file, + const PrimaryKeyIndexDefinition& definition, + const PkSortedIndexGroup& group) -> Result> { + return std::make_shared(positions, equal_call_count); + }; + ASSERT_OK(PrimaryKeySortedIndexScan::Evaluate(plan, table_schema_, PriceEqual(10), definitions_, + reader_factory)); + ASSERT_EQ(1, *equal_call_count); +} + TEST_F(PrimaryKeySortedIndexScanTest, EmptyResultOmitsAllFiles) { ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::shared_ptr split = @@ -419,6 +476,7 @@ TEST_F(PrimaryKeySortedIndexScanTest, UncoveredFileFallsBackOthersNarrow) { ASSERT_TRUE(indexed_split != nullptr); auto fallback_split = std::dynamic_pointer_cast(splits[1]); ASSERT_TRUE(fallback_split != nullptr); + ASSERT_FALSE(fallback_split->RawConvertible()); ASSERT_EQ(fallback_split->DataFiles().size(), 1); ASSERT_EQ(fallback_split->DataFiles()[0]->file_name, "c.parquet"); } @@ -540,9 +598,8 @@ TEST_F(PrimaryKeySortedIndexScanTest, SnapshotMismatchIsRejected) { std::move(files)); builder.WithSnapshot(kSnapshotId + 1).IsStreaming(false).RawConvertible(true); ASSERT_OK_AND_ASSIGN(std::shared_ptr split, builder.Build()); - Result plan = PrimaryKeySortedIndexScan::CreatePlan( - kSnapshotId, {split}, definitions_, MakeEntries(payload)); - ASSERT_NOK(plan.status()); + ASSERT_NOK(PrimaryKeySortedIndexScan::CreatePlan(kSnapshotId, {split}, definitions_, + MakeEntries(payload))); } } // namespace paimon::test diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt index 6c1fdee17..75147ce60 100644 --- a/test/inte/CMakeLists.txt +++ b/test/inte/CMakeLists.txt @@ -50,6 +50,13 @@ if(PAIMON_BUILD_TESTS) test_utils_static ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(primary_key_sorted_index_inte_test + STATIC_LINK_LIBS + paimon_shared + ${TEST_STATIC_LINK_LIBS} + test_utils_static + ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(write_and_read_inte_test STATIC_LINK_LIBS paimon_shared diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp new file mode 100644 index 000000000..8c86f4e04 --- /dev/null +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/test_helper.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +using Row = std::tuple; + +class TrackingLocalFileSystem : public LocalFileSystem { + public: + Result> Open(const std::string& path) const override { + { + std::scoped_lock lock(mutex_); + opened_paths_.push_back(path); + } + return LocalFileSystem::Open(path); + } + + std::set OpenedIndexPaths() const { + std::scoped_lock lock(mutex_); + std::set index_paths; + for (const std::string& path : opened_paths_) { + if (path.find("/index/index-") != std::string::npos) { + index_paths.insert(path); + } + } + return index_paths; + } + + private: + mutable std::mutex mutex_; + mutable std::vector opened_paths_; +}; + +std::vector ExpectedScoreZeroRows(int64_t snapshot_id) { + std::vector rows; + for (int32_t id = 10; id <= 2000; id += 10) { + if (snapshot_id >= 4 && (id == 10 || id == 20)) { + continue; + } + rows.emplace_back(-1, id, 0, "keep"); + } + if (snapshot_id >= 3) { + rows.emplace_back(-1, 2001, 0, "keep"); + } + return rows; +} + +std::vector ExpectedOrRowsAtSnapshot4() { + std::vector rows; + for (int32_t id = 1; id <= 2000; id++) { + if (id == 10 || id == 20 || id % 2 != 0) { + continue; + } + rows.emplace_back(-1, id, id % 10, "keep"); + } + rows.emplace_back(-1, 2001, 0, "keep"); + return rows; +} + +std::vector ExpectedResidualRowsAtSnapshot4() { + std::vector rows; + for (int32_t id = 30; id <= 1500; id += 10) { + rows.emplace_back(-1, id, 0, "keep"); + } + return rows; +} + +std::vector ExpectedPartitionRowsAtSnapshot2() { + std::vector rows; + for (int32_t pt = 1; pt <= 2; pt++) { + for (int32_t id = 1; id <= 100; id++) { + rows.emplace_back(pt, id, id % 10, id % 2 == 0 ? "keep" : "drop"); + } + } + return rows; +} + +std::vector ExpectedFallbackRows() { + std::vector rows; + for (int32_t id = 20; id <= 100; id += 10) { + rows.emplace_back(1, id, 0, "keep"); + } + rows.emplace_back(1, 101, 0, "keep"); + for (int32_t id = 10; id <= 100; id += 10) { + rows.emplace_back(2, id, 0, "keep"); + } + return rows; +} + +std::pair CountSplitKinds(const std::shared_ptr& plan) { + size_t indexed_count = 0; + size_t data_count = 0; + for (const std::shared_ptr& split : plan->Splits()) { + if (std::dynamic_pointer_cast(split) != nullptr) { + indexed_count++; + } else if (std::dynamic_pointer_cast(split) != nullptr) { + data_count++; + } + } + return {indexed_count, data_count}; +} + +} // namespace + +class PrimaryKeySortedIndexInteTest : public ::testing::Test, + public ::testing::WithParamInterface { + protected: + void SetUp() override { + format_ = GetParam(); + } + + std::string TablePath(bool partitioned) const { + const std::string table_name = partitioned ? "pk_btree_partitioned_e2e" : "pk_btree_e2e"; + return PathUtil::JoinPath(GetDataDir(), + fmt::format("{}/{}.db/{}", format_, table_name, table_name)); + } + + std::shared_ptr ScoreEqual(bool partitioned, int32_t score) const { + return PredicateBuilder::Equal(partitioned ? 2 : 1, "score", FieldType::INT, + Literal(score)); + } + + std::shared_ptr ScoreGreaterOrEqual(bool partitioned, int32_t score) const { + return PredicateBuilder::GreaterOrEqual(partitioned ? 2 : 1, "score", FieldType::INT, + Literal(score)); + } + + std::shared_ptr TagEqual(bool partitioned, const std::string& tag) const { + return PredicateBuilder::Equal(partitioned ? 3 : 2, "tag", FieldType::STRING, + Literal(FieldType::STRING, tag.data(), tag.size())); + } + + std::shared_ptr IdLessOrEqual(bool partitioned, int32_t id) const { + return PredicateBuilder::LessOrEqual(partitioned ? 1 : 0, "id", FieldType::INT, + Literal(id)); + } + + Result> Scan( + bool partitioned, const std::shared_ptr& predicate, + const std::optional& snapshot_id, bool index_enabled, + const std::vector>& partition_filters, + const std::string& branch, const std::shared_ptr& file_system) const { + ScanContextBuilder builder(TablePath(partitioned)); + builder.SetPredicate(predicate).AddOption(Options::GLOBAL_INDEX_ENABLED, + index_enabled ? "true" : "false"); + if (snapshot_id != std::nullopt) { + builder.AddOption(Options::SCAN_SNAPSHOT_ID, fmt::format("{}", snapshot_id.value())); + } + if (!partition_filters.empty()) { + builder.SetPartitionFilter(partition_filters); + } + if (!branch.empty()) { + builder.AddOption(Options::BRANCH, branch); + } + if (file_system != nullptr) { + builder.WithFileSystem(file_system); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + TableScan::Create(std::move(context))); + return table_scan->CreatePlan(); + } + + Result> Read(bool partitioned, const std::shared_ptr& predicate, + const std::vector>& splits, + const std::map& options, + const std::shared_ptr& file_system) const { + ReadContextBuilder builder(TablePath(partitioned)); + if (partitioned) { + builder.SetReadFieldNames({"pt", "id", "score", "tag"}); + } else { + builder.SetReadFieldNames({"id", "score", "tag"}); + } + builder.SetPredicate(predicate).EnablePredicateFilter(true); + for (const auto& [key, value] : options) { + builder.AddOption(key, value); + } + if (file_system != nullptr) { + builder.WithFileSystem(file_system); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr context, builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + TableRead::Create(std::move(context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + table_read->CreateReader(splits)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + ReadResultCollector::CollectResult(batch_reader.get())); + if (result == nullptr) { + return std::vector(); + } + + auto struct_type = std::dynamic_pointer_cast(result->type()); + if (struct_type == nullptr) { + return Status::Invalid("primary-key E2E read result is not a struct array"); + } + int32_t pt_field = partitioned ? struct_type->GetFieldIndex("pt") : -1; + int32_t id_field = struct_type->GetFieldIndex("id"); + int32_t score_field = struct_type->GetFieldIndex("score"); + int32_t tag_field = struct_type->GetFieldIndex("tag"); + if (id_field < 0 || score_field < 0 || tag_field < 0 || (partitioned && pt_field < 0)) { + return Status::Invalid(fmt::format("unexpected primary-key E2E result type {}", + result->type()->ToString())); + } + + std::vector rows; + for (const std::shared_ptr& chunk : result->chunks()) { + auto struct_array = std::dynamic_pointer_cast(chunk); + if (struct_array == nullptr) { + return Status::Invalid("primary-key E2E result chunk is not a struct array"); + } + auto id_array = + std::dynamic_pointer_cast(struct_array->field(id_field)); + auto score_array = + std::dynamic_pointer_cast(struct_array->field(score_field)); + auto tag_array = + std::dynamic_pointer_cast(struct_array->field(tag_field)); + std::shared_ptr pt_array; + if (partitioned) { + pt_array = + std::dynamic_pointer_cast(struct_array->field(pt_field)); + } + if (id_array == nullptr || score_array == nullptr || tag_array == nullptr || + (partitioned && pt_array == nullptr)) { + return Status::Invalid("unexpected primary-key E2E result column type"); + } + for (int64_t row = 0; row < struct_array->length(); row++) { + if (id_array->IsNull(row) || score_array->IsNull(row) || tag_array->IsNull(row) || + (partitioned && pt_array->IsNull(row))) { + return Status::Invalid("unexpected null in primary-key E2E result"); + } + rows.emplace_back(partitioned ? pt_array->Value(row) : -1, id_array->Value(row), + score_array->Value(row), tag_array->GetString(row)); + } + } + std::sort(rows.begin(), rows.end()); + return rows; + } + + std::string format_; +}; + +TEST_P(PrimaryKeySortedIndexInteTest, HistoricalSnapshotsMixedFilesAndDisabledIndex) { + std::shared_ptr predicate = ScoreEqual(/*partitioned=*/false, 0); + const std::vector snapshot_ids = {2, 3, 5}; + for (int64_t snapshot_id : snapshot_ids) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + Scan(/*partitioned=*/false, predicate, snapshot_id, /*index_enabled=*/true, + /*partition_filters=*/{}, /*branch=*/"", /*file_system=*/nullptr)); + ASSERT_EQ(snapshot_id, plan->SnapshotId()); + const auto [indexed_count, data_count] = CountSplitKinds(plan); + ASSERT_GT(indexed_count, 0); + if (snapshot_id == 3) { + ASSERT_GT(data_count, 0); + } else { + ASSERT_EQ(0, data_count); + } + ASSERT_OK_AND_ASSIGN(std::vector rows, + Read(/*partitioned=*/false, predicate, plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedScoreZeroRows(snapshot_id), rows); + } + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr disabled_plan, + Scan(/*partitioned=*/false, predicate, /*snapshot_id=*/5, /*index_enabled=*/false, + /*partition_filters=*/{}, /*branch=*/"", /*file_system=*/nullptr)); + const auto [disabled_indexed_count, disabled_data_count] = CountSplitKinds(disabled_plan); + ASSERT_EQ(0, disabled_indexed_count); + ASSERT_GT(disabled_data_count, 0); + ASSERT_OK_AND_ASSIGN( + std::vector disabled_rows, + Read(/*partitioned=*/false, predicate, disabled_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedScoreZeroRows(/*snapshot_id=*/5), disabled_rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, MultiSourceOrdinalsBecomeBoundedFileLocalRanges) { + std::shared_ptr predicate = ScoreEqual(/*partitioned=*/false, 0); + auto tracking_file_system = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + Scan(/*partitioned=*/false, predicate, /*snapshot_id=*/2, /*index_enabled=*/true, + /*partition_filters=*/{}, /*branch=*/"", tracking_file_system)); + + std::set source_files; + int64_t selected_rows = 0; + for (const std::shared_ptr& split : plan->Splits()) { + auto indexed_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_FALSE(inner_split->RawConvertible()); + ASSERT_EQ(1, inner_split->DataFiles().size()); + const std::shared_ptr& file = inner_split->DataFiles()[0]; + source_files.insert(file->file_name); + for (const Range& range : indexed_split->RowRanges()) { + ASSERT_GE(range.from, 0); + ASSERT_GE(range.to, range.from); + ASSERT_LT(range.to, file->row_count); + selected_rows += range.to - range.from + 1; + } + } + ASSERT_GE(source_files.size(), 2); + ASSERT_EQ(200, selected_rows); + ASSERT_EQ(1, tracking_file_system->OpenedIndexPaths().size()); + + ASSERT_OK_AND_ASSIGN(std::vector rows, + Read(/*partitioned=*/false, predicate, plan->Splits(), /*options=*/{}, + tracking_file_system)); + ASSERT_EQ(ExpectedScoreZeroRows(/*snapshot_id=*/2), rows); + ASSERT_OK_AND_ASSIGN(std::vector ranges_only_rows, + Read(/*partitioned=*/false, /*predicate=*/nullptr, plan->Splits(), + /*options=*/{}, tracking_file_system)); + ASSERT_EQ(ExpectedScoreZeroRows(/*snapshot_id=*/2), ranges_only_rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, DeletionVectorResidualAndBooleanFallback) { + std::shared_ptr score = ScoreEqual(/*partitioned=*/false, 0); + std::shared_ptr id_upper_bound = IdLessOrEqual(/*partitioned=*/false, /*id=*/1500); + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_predicate, + PredicateBuilder::And({score, id_upper_bound})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_plan, + Scan(/*partitioned=*/false, and_predicate, /*snapshot_id=*/4, + /*index_enabled=*/true, /*partition_filters=*/{}, /*branch=*/"", + /*file_system=*/nullptr)); + bool has_deletion_vector = false; + for (const std::shared_ptr& split : and_plan->Splits()) { + auto indexed_split = std::dynamic_pointer_cast(split); + if (indexed_split == nullptr) { + continue; + } + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + for (const std::optional& deletion_file : inner_split->DeletionFiles()) { + has_deletion_vector = has_deletion_vector || deletion_file.has_value(); + } + } + ASSERT_TRUE(has_deletion_vector); + ASSERT_OK_AND_ASSIGN(std::vector and_rows, Read(/*partitioned=*/false, and_predicate, + and_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedResidualRowsAtSnapshot4(), and_rows); + + std::shared_ptr tag = TagEqual(/*partitioned=*/false, "keep"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_predicate, + PredicateBuilder::Or({score, tag})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_plan, + Scan(/*partitioned=*/false, or_predicate, /*snapshot_id=*/4, + /*index_enabled=*/true, /*partition_filters=*/{}, /*branch=*/"", + /*file_system=*/nullptr)); + const auto [or_indexed_count, or_data_count] = CountSplitKinds(or_plan); + ASSERT_EQ(0, or_indexed_count); + ASSERT_GT(or_data_count, 0); + ASSERT_OK_AND_ASSIGN(std::vector or_rows, Read(/*partitioned=*/false, or_predicate, + or_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedOrRowsAtSnapshot4(), or_rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, PartitionsBucketsAndIndexPaths) { + std::shared_ptr predicate = ScoreGreaterOrEqual(/*partitioned=*/true, /*score=*/0); + auto tracking_file_system = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + Scan(/*partitioned=*/true, predicate, /*snapshot_id=*/2, /*index_enabled=*/true, + /*partition_filters=*/{}, /*branch=*/"", tracking_file_system)); + + std::set partitions; + std::set buckets; + int64_t selected_rows = 0; + for (const std::shared_ptr& split : plan->Splits()) { + auto indexed_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_EQ(1, inner_split->DataFiles().size()); + partitions.insert(inner_split->Partition().GetInt(0)); + buckets.insert(inner_split->Bucket()); + const int64_t row_count = inner_split->DataFiles()[0]->row_count; + for (const Range& range : indexed_split->RowRanges()) { + ASSERT_GE(range.from, 0); + ASSERT_LT(range.to, row_count); + selected_rows += range.to - range.from + 1; + } + } + ASSERT_EQ((std::set{1, 2}), partitions); + ASSERT_EQ((std::set{0, 1}), buckets); + ASSERT_EQ(200, selected_rows); + ASSERT_FALSE(tracking_file_system->OpenedIndexPaths().empty()); + + ASSERT_OK_AND_ASSIGN(std::vector rows, + Read(/*partitioned=*/true, predicate, plan->Splits(), /*options=*/{}, + tracking_file_system)); + ASSERT_EQ(ExpectedPartitionRowsAtSnapshot2(), rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, FallbackTableReadRoutesMainAndFallbackSplits) { + std::shared_ptr predicate = ScoreEqual(/*partitioned=*/true, 0); + const std::vector> main_partition = {{{"pt", "1"}}}; + const std::vector> fallback_partition = {{{"pt", "2"}}}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr main_plan, + Scan(/*partitioned=*/true, predicate, /*snapshot_id=*/5, /*index_enabled=*/true, + main_partition, /*branch=*/"", /*file_system=*/nullptr)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fallback_plan, + Scan(/*partitioned=*/true, predicate, /*snapshot_id=*/std::nullopt, + /*index_enabled=*/false, fallback_partition, /*branch=*/"fallback", + /*file_system=*/nullptr)); + + std::vector> routed_splits; + for (const std::shared_ptr& split : main_plan->Splits()) { + ASSERT_TRUE(std::dynamic_pointer_cast(split) != nullptr); + routed_splits.push_back(split); + } + for (const std::shared_ptr& split : fallback_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split != nullptr); + routed_splits.push_back( + std::make_shared(data_split, /*is_fallback=*/true)); + } + ASSERT_FALSE(main_plan->Splits().empty()); + ASSERT_FALSE(fallback_plan->Splits().empty()); + + const std::map read_options = { + {Options::SCAN_FALLBACK_BRANCH, "fallback"}}; + ASSERT_OK_AND_ASSIGN(std::vector rows, + Read(/*partitioned=*/true, predicate, routed_splits, read_options, + /*file_system=*/nullptr)); + ASSERT_EQ(ExpectedFallbackRows(), rows); +} + +TEST_P(PrimaryKeySortedIndexInteTest, ScoredSplitFailsBeforeForceKeepDeleteFallback) { + std::shared_ptr predicate = ScoreEqual(/*partitioned=*/false, 0); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + Scan(/*partitioned=*/false, predicate, /*snapshot_id=*/2, /*index_enabled=*/true, + /*partition_filters=*/{}, /*branch=*/"", /*file_system=*/nullptr)); + ASSERT_FALSE(plan->Splits().empty()); + auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_TRUE(indexed_split != nullptr); + auto inner_split = std::dynamic_pointer_cast(indexed_split->GetDataSplit()); + ASSERT_TRUE(inner_split != nullptr); + ASSERT_FALSE(indexed_split->RowRanges().empty()); + const Range& first_range = indexed_split->RowRanges()[0]; + auto scored_split = std::make_shared( + inner_split, std::vector{Range(first_range.from, first_range.from)}, + std::vector{0.5F}); + + ReadContextBuilder builder(TablePath(/*partitioned=*/false)); + builder.SetReadFieldNames({"id", "score", "tag"}) + .SetPredicate(predicate) + .EnablePredicateFilter(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(context))); + auto* key_value_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(key_value_table_read != nullptr); + key_value_table_read->ForceKeepDelete(true); + ASSERT_NOK_WITH_MSG(key_value_table_read->CreateReader(scored_split), + "Primary-key reads do not support scored indexed splits yet"); +} + +std::vector PrimaryKeySortedIndexFormats() { + std::vector formats = {"parquet"}; +#ifdef PAIMON_ENABLE_ORC + formats.emplace_back("orc"); +#endif + return formats; +} + +INSTANTIATE_TEST_SUITE_P(FileFormat, PrimaryKeySortedIndexInteTest, + ::testing::ValuesIn(PrimaryKeySortedIndexFormats())); + +} // namespace paimon::test diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/README b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/README new file mode 100644 index 000000000..4f86a4af4 --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/README @@ -0,0 +1,24 @@ +id:int score:int tag:string +primary key: id +no partition key +bucket count: 1 + +Generated by Apache Paimon Java release-2.0.0. +file format: orc +deletion-vectors.enabled: true +deletion-vectors.merge-on-read: false +write-only: true (explicit fixture commits produce snapshots 2, 4, 5) +primary-key btree index: score +target-file-size: 8 kb +compaction.force-rewrite-all-files: true + +Snapshot semantics: +snapshot-1 APPEND: ids 1..2000; score=id%10; even tag=keep, odd tag=drop. +snapshot-2 COMPACT: full compaction builds source-backed score BTree payloads; source-file counts per payload are [2]; 2000 rows. +snapshot-3 APPEND: insert (2001,0,keep) and (2002,5,late_drop); indexed compacted files and a visible unindexed APPEND-source file coexist; 2002 rows. +snapshot-4 COMPACT: delete id=10 and update id=20 to (77,updated), then lookup compact in one fixture commit; deletion vectors coexist with indexed files; 2001 rows. +snapshot-5 COMPACT: full compaction rebuilds the BTree source groups; 2001 rows. + +Fixture construction note: with deletion vectors enabled and merge-on-read disabled, Java and C++ batch scans skip literal level-0 files. Snapshot 3 therefore promotes Java-prepared files to the maximum manifest level while retaining FileSource.APPEND. The source-backed PK BTree policy indexes only COMPACT files above level 0, making these files visible but deliberately unindexed. Snapshot 4 publishes the Java-prepared compact outputs and index changes while omitting their transient, never-active append inputs. These are test-fixture constructions, not ordinary end-user write workflows. + +At snapshots 4 and 5, predicate score=0 AND id<=1500 returns ids 30,40,...,1500; id=10 is deleted and id=20 has score 77. diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-5ee15f36-7a45-4651-8c81-a55e49dbf9ff-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-5ee15f36-7a45-4651-8c81-a55e49dbf9ff-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..e47187451a2f8d295a7909e56650dc767a78ca2e GIT binary patch literal 757 zcmeYdau#G@;9?VE;b012Fa$EixR@Cj7=-van1nbv7Rd`Nk(W4Thec|Eyuc!Pi9?=sC*7u!Wh6je`y7Ty8MUC?U6krr!l;8vkuz>^wfC5YcY!Ylj zOdJeA%*4dQpf7XhDZ{2SXSi=Ml!v!8pGjNBklV0ZZHnqOs2z2p%nU57Ts2(a;N=oT z3)BcsXeja_hg$?2Ff62TdOkvojlqFOL18ts#8hpG>eJ68G?>6XN+@LrRkBUnJLyLl&f>r82f0I9NLxN8(s5nv5=kWSf=$wKdY1$pKQ5~vV=lzkr z*IJM&c+B_`yFf}}!jgsOq+4tecq;Q|E}Oou<#n{zCe@;~ z6Ec@9Z1UXZwodhv@H+Vq7dO~_3H4cQb5HVHgx&87d7JpR+~qfI);%`j{gstJYkJ|` zYl)Jrr$2wXP%`I9+9D3S-&e0GSclD)@jd3gi;WOKW zZs9$P;$!FiZA!Ot^%hKdt(@3!{7qJXn97s(C%4wWs49BP&W#il(z9nW&)qjO;eaH= j73q>wJRH-R1Xv{+8U&P>7E@*!)MTIqhOw8{KfzM=kr_S_Y^Ix?5J$`eroGJ z_v`=U%v~!!cJ##<$30rcqc666b@suNSGV#o=u7W8%{XiJOBR7-)#G7r84Qg-H<=xX z6+R{O0BU-(Ei;1uytF?Yt7x z$9vf1oM|8XV-c$#$DEctQ)){8`k-dEqG|WmsDgbHor+w9vlUInPh8An5I*-k^m6%v zDd8*(F|rJrQ4fmxXZQs%K|-j8;V%b6hyVw(z&vItULduVQKExUAx;Ar9ZnoPLTrpo z%q*;IQta{Gu95MXDN+LQ!LEU!u71w0@qVGcPOd>x-0@+KKB2Dh-kyFgQcOT4ti{Rs zMX6HEC5h=u(j3fI;#`~siJ7_id3u>CLR^Lh=EmkGh89M~KuH!OLlbio3u7}AGa!%A zR7nV^o{!PM5XfLRGHBqif+ScgAyCRePNoq&uw=;(l7gqi2w`xFMkKNb4J!%ki7P_W zVS=9rLj+?ItD&Byo&_f~d_FNgbXcCRCQ-n~V9Gw%h?!#zlK`tkLxX@469bP%u>oh( L7iMPvAZG~x3P|9@ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-1.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c0e6caa4-c74e-4a9f-88b1-9170afaae1aa-1.orc new file mode 100644 index 0000000000000000000000000000000000000000..fa46994ff7a7719365bcdd5f089f519cba37927e GIT binary patch literal 874 zcmeYdau#M_;9?hI<+#8ez+et!OL8$YFfa%S0R@D(Ihq6n7RXCnn7M@!n{0!Cz&v@0 z4P78vU7%hOh+bZhUN#N}0R{;MY}$|w)dy-5g=pggX=4X!6Owq~62M>&l#zwVhyi7k z__!X9b z7uW=tKSf1P`v5h)vzD2`=$N?T0!>-|KkQ#O3H~_2blZddc!3pL;*prd1Z`ufNs&uZ zw+D0lQ&Vc4*M3mtN`cOdY@Vyj${(zMw5W6YsVrxv`t`@76;`k947oB>!HxN!TKAHx z3l;2qCR`2Mk-wEu&ry$s!Qn+iP)g&OiAQRfBA6f{)x;pm$xtG|!7Q+xS&A1(?Piqd zW>kpN0LF|H2agaNBNH&6mv;px{@>pvz0g(XF+0SZhoF#W{MD(p@F%vxrw2Lkugw` z#mLaa+{D7z%)|`HV>DF~0;=a@G%y4**o_P>uv`(5Oh%6N bOaiPD4GjWHObk34)drkRUznNwgPbJ*Zwm68 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c917f42c-c01c-4d16-9c2a-cb89c60d8d6a-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-c917f42c-c01c-4d16-9c2a-cb89c60d8d6a-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..2bb75455368ad9e1904da1be645dc3dc6f227d96 GIT binary patch literal 764 zcmeYdau#G@;9?VE;b012&;~MvxtJLk7=(B@n1t9kLYyubzf#PBiaekmUC&v zJCg(_vjiIpE1Lu#P>2cSDIq2f1|Vi);$hI2x$|_xq}24d3v3e@&o!HAze}@XJ90Dh zz%!^l6`{-wTUfcuxWM7c1@bC9G$Yu#1kpkuksmx zo_R7v(u4`@yM%m(Rt|Fg^+>T))wQ_3|_BBw;(TBl)J@ z)_IKU-2(5usny#fep&rY*Tqx!7-P~VWKKEClG@BZDS`2+_yp&PB3$nBqH>*!j+KO( z{cVgqwYlV}{jz#p4g1+|r;2IVvrMWH+&O*1c7wlO;f9RE3GOg-(=3VQnR@|We*F^Shz?hXQae3#n$E|w|Inh z%P!*P_i9(Z$}{iq^5P54Os?}YINvogRL<3}kP$hk{_E<)8`?jXhu;bcUV1sH;d@Tk zNreZWWOja4pEd2yFGfySfGiA*WW2k3rpbX^hAYw~r+7H#G6}FsG&Be(F){FHlo@a~ MePL$y4|0|O0FW)rHUIzs literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-e165b892-027d-4e7b-9750-3226cd0410af-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..db298b1680933d0870af394139afbd47a6540e15 GIT binary patch literal 1543 zcmeYdau#M_;9?hIqh*L4ZMm0h=a-frda$Vh~OIAWa-VO|uv!rg#Q0xB{gVAX1V*DJ3B; zmXxCW0wMO&f|SIP)D(#}prsZJ3>sVY|0;COWnjoS{GW*_G%PgW;Qtm5CPoHJR=2q< z3={t|J_h>5!NEX5gomAlsWIU|hJ~B}N6W56hZPZe51s_nn6V?yK%%ED*>S~)y=;xU#C$Yi-o}h&9(% zm!5iCwLf9QjWuQ7y0gw_Y`L|z>~!v~`vp7htSk3kd+UA0o_p)dPrv>3zu~}x4HZ6m z()=Aq9&M~RlV>eI;lz_ol|JjD^=F)Uwz=}myK4Id7hY_s^3|K|zv9ZPtyO39cE@kH z@n&1K@4DOhJMO&OUVZl6@A?A|KJ2LR)0gf)@#NFansfQq^Dn&kva8l_ef0VpZ@%rW zJ@>wP|AP-d_SE_7&p!X+%dfq4=ks^p|M26_zIy-lx8MKx^KXCs`S-v7F$n!X_vGOJ zCJrV=28MbRm`^AP=0H+t0tZtnBZCN2`+ufJ1|gHMiiygu>75rr1mhI{{~TOAd;&rY zOe}0n44-vhJ!cD@VFgXvN0u@$L^%H!6l8JnaB=*i1ZcHu(@(c{1 zY>}OsS^&)$SD+fF^D;98u?k8YP?h};1n+;9I%>~aT2<__Y0KqlL8oM=J=2~psh51u zMEvyZr;8q_&Y2@$xciXyN!Ih4CyyTFw%GY7tTFi7(TB>fleii5rT3g>oW=Yo?A*ee zdIEWATo>4OWIIJNeSuom(aX$msZCOOfu`*Lh)}^Bv+t*MH~)JYBY5s^BcJT7zDk$O zMV{XEMI4@t7e1}(FmhEssMM~on6=q-di!>1zNoks7jdp%=eLW`dAe2Vufx8%1-_}3 zQ%znfFPvLFE!FR8Jd`zSQwj`m;rf=rb5f!zUobz{JAF!R3E}fs2XZ%uny5pF_QcOd#qMcJ?wbI5_@a z<$S@#wSkF&e@U~D1QRC%gBz2}1Z4&Wkc+ZYQw!kE+X2-#Z7vhTr50X3hK*J$S1{eM zuR2jPVb{y)>B(jxIUNQ5V4riz{=c`!sUCb1go zS?XDE!oq3tXs_`^{wJ*uo^hD$&p&pv1($qtVQ0z}fVLnb|+cSrPy_ CpzK}% literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-ed9149c4-c451-4780-a2cd-fe8a74ee2558-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-ed9149c4-c451-4780-a2cd-fe8a74ee2558-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..d2788f2c56da4fee1a7d0a77bd32a9c813150697 GIT binary patch literal 732 zcmeYdau#G@;9?VE;b074&;~MvxtJLk7=(B@7=_q4Gz2sxGy)h5f#PBiaekmUC&w~* zfo1X%%jB_WVGv-DV89}d%cxn50<#z;W&v&Y2HK*`#R{|?XulK(qml?0duc&RVo7R> z5R4(g7r?;Iz~INA!O6f7$H2b~sQf=v1S~#_pMe3Y3CMS2C^*5ups`i|uR_mVCWb4_ zTx=X{KnHSzX+{YNCJAX~7FIS14WIy{0HXvOlLR}n1S?303n;`Sz$U>a#Kge>#7s;) z4Ei#6o^F_wnjUw7ZNkQ)+VnK-$h2pyxibQ$oq<|Y^OlK0=%|qNhNYHQq&k$FiaC$j z&9M32aid)QQNX6woSsy!ThBLVu(CC%D$KB%$;{6^vSA})D# z=NpXy$2G&QPnyxoz4PG=jm9_r59Yj{^J-4UoJV^$pUa%{=<3e!<#BV{BmOf4NG!TXzi9;Obb=Zj;8F? zEJ^i`=u-H#BZYC^B-zs!*#+ePb8CICTm?FtBkx776 YqM<=RiHU(nBh!Gh=?gQne~_~T08X6fKL7v# literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-0.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..16bc68db9116ef42a44ee8ff0fe5563305392666 GIT binary patch literal 854 zcmeYdau#M_;9?hIE@*!)MTIqhOw8{KfzM=kr_S_Y^Ix?5J$`eroGJ z_v`=U%v~!!cJ##<$30rcqc666b@suNSGV#o=u7W8%{XiJOBR7-)#G7r84Qg-H<=xX z6+R{O0BU-(Ei;1uytF?Yt7x z$9vf1oM|8XV-c$#$DEctQ)){8`k-dEqG|WmsDgbHor+w9vlUInPh8An5I*-k^m6%v zDd8*(F|rJrQ4fmxXZQs%K|-j8;V%b6hyVw(z&vItULduVQKExUAx;Ar9ZnoPLTrpo z%q*;IQta{Gu95MXDN+LQ!LEU!u71w0@qVGcPOd>x-0@+KKB2Dh-kyFgQcOT4ti{Rs zMX6HEC5h=u(j3fI;#`~siJ7_id3u>CLR^Lh=EmkGh89M~KuH!OLlbio3u7}AGa!%A zR7nV^o{!PM5XfLRGHBqif+ScgAyCRePNoq&uw=;(l7gqi2w`xFMkKNb4J!%ki7P_W zVS=9rLj+?ItD&Byo&_f~d_FNgbXcCRCQ-n~V9Gw%h?!#zlK`tkLxX@469bP%u>oh( L7iMPvAZG~x3P|9@ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-1.orc b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f663e836-b9cb-45fc-b513-266c4b33026f-1.orc new file mode 100644 index 0000000000000000000000000000000000000000..fa46994ff7a7719365bcdd5f089f519cba37927e GIT binary patch literal 874 zcmeYdau#M_;9?hI<+#8ez+et!OL8$YFfa%S0R@D(Ihq6n7RXCnn7M@!n{0!Cz&v@0 z4P78vU7%hOh+bZhUN#N}0R{;MY}$|w)dy-5g=pggX=4X!6Owq~62M>&l#zwVhyi7k z__!X9b z7uW=tKSf1P`v5h)vzD2`=$N?T0!>-|KkQ#O3H~_2blZddc!3pL;*prd1Z`ufNs&uZ zw+D0lQ&Vc4*M3mtN`cOdY@Vyj${(zMw5W6YsVrxv`t`@76;`k947oB>!HxN!TKAHx z3l;2qCR`2Mk-wEu&ry$s!Qn+iP)g&OiAQRfBA6f{)x;pm$xtG|!7Q+xS&A1(?Piqd zW>kpN0LF|H2agaNBNH&6mv;px{@>pvz0g(XF+0SZhoF#W{MD(p@F%vxrw2Lkugw` z#mLaa+{D7z%)|`HV>DF~0;=a@G%y4**o_P>uv`(5Oh%6N bOaiPD4GjWHObk34)drkRUznNwgPbJ*Zwm68 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-2e50e625-15ac-4994-a320-064d8e34d028-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-2e50e625-15ac-4994-a320-064d8e34d028-0 new file mode 100644 index 0000000000000000000000000000000000000000..77fc5c5306a186e43f9ff59158d372c80a2bfdd4 GIT binary patch literal 4083 zcmZA4Q?&QZbqDY>_WbHLRcb4>k@79mHZ@bbO{$bRwr%_Tk8RtwZCgEW`dno$X6-dI zcb~QP>@RkETfV3dG&*6(2R*`bxC@uzBppAUhp0)f+Twiwn2pT zv(M(z?3-2dW#!Ja88q#t))bp8lVHNk*XKDt=24&PUA?R)b-!-aA4+kmb&<}{aXLhM z>3emmekjV>R*Py}^{Qr7sR~q@`sn~CMg=Jkbt{kMrkt0f@>gDMw``QbDM z#qWq~aVXZstQZ!ZqF$7WKkasMM3RUQe&U&5@_oL_|Nh!e^FiLuYk4ux;t4#A|NEi! z;g9T`jXJw*nN6~O<^k4iE30NjEQ7_d5az}1X%LvTr*xYx(s9~Ln`tF2pfOv1J0wWT-*C*cU} zho8|U+DFacb63$c8bs}=78Ro`lz@5x25%TXZy0X!^S=BWzVnBF<|DuRXaDd?PY?`Z zK{_Y|RiFj*feEk#cEA~U0NyYZ#=}hLf;F%W4!|k60{7qre1g6x93`S`RD$YI2O2^% zXbl~pEA)c=aU@R0xws5B;4VCZ=kNwT!Z+BR1d?cyO7cksX(Byjj4Y5XazgHiCk>{t zG@TaGD%wK(=mcG&JM@e`P;VB>;#nqhu^QIK2G|r^VHfO)`SNg{$g_C~uj3tjh|lmf ze!#Ez3-=e1B3a~$GSMKq#E6&^8{$aZ2zMDMqh+ehmld)}_Q)~0Ah+a+ypx_PSjDPz zRj8^|i|SJoYDw*=Gxeanb*PTlncAgmbekT~Q+h@3=?nd&eNDJYG})%a)R_)5WM<5o zIWSk|#rWGun{0D!nQgFLcErxv4SQs7th*EFL_4WYzEj~eIX%uE!|E0GS~&PsNAO5 zW+%$Ff_0RH0_>}~HiziYteaW$!D}>ZI!&W}MVGMNR63<5#{}Rc6Jh+!K8(WO_^emq zrB1>7dR6D)XT1xg%W)2>)!$XD8*!FS&|%s~cjE~3sLoZIGm59-uKJN>HL2!t zziL(0s@rbjL0F_R)G=;{aq4G7)Gdy2yi^gmmmZ`Poyu*wD1*qj?3KUTEMtfts+0vX z$VnrUAWg=|AXz{>I`8vO~r(~A@<1knEq?6b4Qtm~wU=B~>5&ZK*s2~52XLfAkXac-s zAP!*k&SHa~*i0G@*VqEqGi(SBu>loNU9?TLkPcR(ra&F@bQWxnsG(iR8;UzJPc#!;&=cI^?9n+M>rBx;5(-vmsCoh=&;>3) zz8yhV@P*ysJ>ZX4KpRN4k$evQoVThFUl45{yXGq7& z4qe8BrMsxWp0Z9S!a`D_>)>~1>l@yLbJ3kh)VVxd?+{;oA_hPY{{1I)#nPP%RYi{Y zp4t*)*qbhhJG-LB#E(uXjNe(bI#DPE<)KPjjv7%nY6qie9&Msybc;N25N@_HxE7@0 z0$hokaW5Xni+CFsgHzmV@39vNA#o&w6p?C@1zJfznIz+One38t@<@DW0tllCG>aC~ zTDoZ4=^&k^VPKW+(@ScesUM4ANo?EZFdtCL>RBfnX0vRa9kTzrX0I%OpV}y%!t;1J zZ{#2E=A(R`Z}MY)Z*RGW2oir8Bho~Hs1(hjSBwiUXHopYwm22{!b^t8IGG`f8j7VHoA z$b?PScRZVZZ={}s;q<{~I{rErdJ2E`qViE6{;N55r3!5z4;Jp|KuzEQXHEV0l3gJ) zDptJEA+>~myhDw^Iu(sdK)M*BC29x9JJ~8xUC}9Wh6}|34Ob7?S3OBg-&v3-c+8UM zE%V?JbQT2h816^Y___5!1^k~Yc?xXi&!m?pfpLCG7Wt&z=KJK7H=}$0*Iwe$=D`rL zO5#L@m?lM{S`3m_(M~!+zxbm`Q4W_yE!h>0k8>`H$)oU*IUr0{q6GOHStLsq%SM<$ zYGu0&BZD%`>9^BzRqo4Q@FAD-S$0D|^@to)8RR!qpi_%$6E1ribI!~ADYTT%cY_}eCMs*Qxwe$L0H+2Tygva_; z7dvs-!wiEU6JtVfnn^?TAP5zhudXy+_!u^uUUQGe&7!GwPSLhGHTUKNUUnPZ!Vnv0 z7tuO61mox$WY||QOV{bLJ*3xknD(Mq8UX@W6!Qa}GytZsJXTN3*{*G5rL>z>+fg>p za_A=eAI#oa&=)yEd*Bn!wEoH^##p4fz&A8m?cqW+LvmFg$`obl!wo9YS;6kCOT~*3 zH3ozEoO-fTI3I4P&k7Ysstl}=Kz5@Bu(xp6-|r6sb+mrqU9=5HQcsks7htyYH_;KEyMOn;jvUtpsldy=@^cqbKZNLhT()rtx+OrSl%P0VdE9@TWEQ22{X3u_7*n zuTw(YVI6rAzOo9f*l_uq*<_ARi9{L8V^Oy3aB{&0Es=?2Ks3P)QYQy&hy3Gk5)FrB zo9F^FGTb?$Yw{O;$$_j9S8|A8{?5XW43lNfcKIlswy$y?GoZYiU)@sVtZw*>6^N6zWEIEZCJcBo?L;0y%yNWxVXM0FG$*S#E{brIUIUX!Q zM1dj{05-v>YDFHfmFLL69ft+5fQ6uW<>ySANi>Z+Wr%46uVhOp72^&3%Ly9~8UBL)Vip=?1l;9E|JFKBl)I zNMDLLlLDGqJ1&)vyqlhz7~QMCcHh=Jr|jD+VIJM2X?k7uqieD++U+8YkdGz~73gYI zPIH`7U5j_oIgep+yi(88Rgqze__d61+J%ozq0Kr21yC&BEZz@$fwTl)i9Vf)M&yL9 zLBG7mmvlCM5xS9lXW(=%d$CgKt2NIvKt=r6s^6!&++aW2WWq2}vO zXgVxpT_oO=Il=ITMAJx_X?EBW^Tm0C##}%48GRLoIBJKA8q*L%O)HeZeE-Pux|V zZIC5m%?7G}yTW}i+(xSvbb!aei%nHObw@LKAc!RSDqAGlP~HM7R4Scdg>VhektX#3 zQ$deP6rOBMZGmK`%b7w4c0tYH30Pu7Q5)J)Gd5gw$$WaEe((eg;XZb!>V&U|N5L$f zdun&kVfh9L);+WXKk+*ds~5lkx{=;|O{D7%alsEnp{@XRcnMbNTql;5kt%q`TJ(3k zQMiS?1-=iw4YxUOL)C}gK9J_O#`WGC(zJAQ`1Y>bJ0(J;?sP{E$@Tx fGrm3k_^SPM@fn-T`Tv8@ynH`gzW31&f8_rF-ZJAA literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-5890ac3e-74e0-4d3c-bddd-542207d3a13f-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-5890ac3e-74e0-4d3c-bddd-542207d3a13f-0 new file mode 100644 index 000000000..6b2aaa764 --- /dev/null +++ b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-5890ac3e-74e0-4d3c-bddd-542207d3a13f-0 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-7d01de51-d440-42c2-a2ac-f8b89d8e9d36-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-7d01de51-d440-42c2-a2ac-f8b89d8e9d36-0 new file mode 100644 index 0000000000000000000000000000000000000000..07832286cce284be239917d0944a4aff7590990d GIT binary patch literal 33 jcmZQ%U|^7lbN+O}%7B4^5y%0N0zi_JL6{*lIyn~rP%8wL literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-a3ae7e50-6220-4663-89de-4b9b31f41a39-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-a3ae7e50-6220-4663-89de-4b9b31f41a39-0 new file mode 100644 index 0000000000000000000000000000000000000000..c9776973c9c01bca6a2a344ca6d39777fde2840d GIT binary patch literal 88 zcmZSaWnf@nVr1X|QVa|=Yj@hPfJGR1AtFj=j!Xngp%Fq*J};U$BarVI;uHV?Rn`V| literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-c940705f-8d27-40f3-a9ba-a896eeecd29a-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/index/index-c940705f-8d27-40f3-a9ba-a896eeecd29a-0 new file mode 100644 index 0000000000000000000000000000000000000000..025c6a081174891ce38b2f0d25ee553584781361 GIT binary patch literal 4096 zcmZ9PRk*NSQHJ;GUN5#-Ad1+EjfH@eh@fI8Vs{tD?(XjH?(XjH?(XjG{^7#|=HQt* zm^r$yDeUrU-lzuzBVQ+6mO8%jR6mP$d?`l3wvM-pdS5)ERWYuo#b3{ZQ2T5LMKNv{ z-nLfs>SD183Q@EE$xKnH<3+m-75{n+>YYpLEwY_Qp0CgOquJ%RAXR7LXuZtO%_I-h z{oEC}!*jH6Tluc3=CP=dyX$lw2$#(_-Ktpr^TGUBdGcKp4gEnpzGsQfDce-bXq(Ne zR5)oy)gntrs;k!qgS+Ft0drtK&#G~LuyHPT#APRC(CN;lIumtMdD-18x3qsHg9>N{C4#%N7TY|H20?t5T*attM z3si(k>>gS{Q>X?Fpf;3c-`n6M*f8s3pUh8Pm_4&%K3wKZnE}&gYRsoZk;yP|=C?wO zmwC`<`g1#aNl)m%$#MF0i>}fI`rp!Yj1JOnAj$F2cj`oKsQ|E`##E1r0ufG=s!(|< zMWui!6`;{=5BczL*!3I)Ba>xUGPZEhBe&hvvp(}iV*KiNO@C}Py2IN97!1!&cZ2C*d;Oh3D`Q zdZSPjk1|m)szvQ+5KW_1w2v;)GxEjZI1y*#Qe2Na@i3mn>-Z2~V^`u&B1tmICFP`% zbdymsPd3Rhxh3v2kVex~nolcfGwr40bdhe;Q+iK5Sul%b>8y}dvsTv6CfPDOXOGOA zhw^xy$%}a{Z|8%2ny>PGe#xJ?uLu{3B3qP-deJF{#jIEthvHheN`DzClVz?fmyNPp zj>>ttDUao?bXS2YTBWLdRjHa)uNqg2YFnMEd*!Kvb*xU;g}Pd|>V7?`m-Vhb*N@uU zgqnDhX^Ks)X*Yvr+N_#=b7`K9uMM|}HrtlkdfRD-?W|q5hxXdKI{r?ilkDU=;v^;A*=?*u?$wkieL?EV*{+mnPM66$gZ$GcERG{ z6Z7F=JOn22EMCIvxEJi;H+G26@CUHQ5BL>-;b*{4M2IA@19F6$QzjZjmskQLVoq#` z32-F7=0>>50NDouoG6(h^JELCkWI2jj>#&pAh+a+EC5l?o%B#aDn_M&G*zIgR19cQ zeQH9bI7=!B?5H#Kpge$=4$*NsL;pvSuF-i;oBq22J*8Llp8k^y{iKzT`JFJ6U@Dv} zQ(}I(&UBa|^U)czW)94i`5E|PnjAkHVShNu=GZdZf$Go@nn7!*!5*LzbcJ4!AI?G% zI0@(A?<&Iyr~!B3E<1uT{!(9nWbw88td&Lzf8i?5fhUcJ$F?3_vS#O+AM$nn2xj># zdWO;DnFW$tJIuekllMCH+#8m1U%m>nc_I&m;d~ekLU(eD`{_OLMaOs(&trevZC&lP znWWS3XAjM~nKkA3WA7X`%k-3Vno(SD5>cs%rIpUF?9ybLZ4wO%H|;PN7t-H7r|Hz! zgrjGDsrU7tmfLOeNcZ8Yp4Njp-$}-QziHdGH(MmNx>)N>Jx*%jkNjsS>m~6zRD0`4 z>gGtq|T9rS}WVtpLRI5Uju3}X+O$WiMn9W;H746(hS6mDW z!Koazwd@jZ%SAaZt3j`9mX)#$oDGyGDnyf50Ul0|C_`gn zK}>-jyd{p%iMSIUasUL$7?~#9K#)@)Z%~zNk$tiTOvokKWOrl{IFoMpL3*heCq!j{ zIF*AkRFR4UH7WqNsR1>mLcogJQ)$kH@&Xa)Nkw5F9i|iX=S`L_(RKPCI&=yi(gn_p zUekYeps)0c{yjgFha*gq$uYlLW|B~q(_p&H$H(x9slan)!yK9S-xxQ$fO~9!-NGl> z58Yu85=36$kH(NRQb3|?6}f^fWX$%F3333N@DgcpcE}k?u@7VoeybP7AvBI+`ld5B zQKjwF*{~kE+hsDWZ}~Wmm8p1E-1E`Kf=HDgzE34Xtr zZkkK#Z`#2+EF_h9mqeO*)5-F2vMH3G*{8Y2(-}42+X>6fIr)WdbIeR!Fe&GGB6Q`?U|9~sMNv)a*{&Sr#b6(t%W2Sv zy7@YD$Cctyrjtfq%4(^%$^_eD(T1w8Zx*r6sV#-ABwo!jPm+tud9=t>skYf3vSM{A z;(;%$Rrw;9XERrnN&R`diX}D>AEQB)%tOJnx)-CUpG5Li6)3`CB8x=(Y7pGxZF{MP zSv>Uy>CU*#x6jH`4$@HSt55MPEJq|7Pm*x`Ux7B9CF*{s*ZQM&dI+=iHV&4hdJ*@E zN8&DgS-tMX|A^Te787;?NAWxUAi8uN{qY>#06pxJ%z;~|iI3<3GRN*r8CxM9_=U#F z5jdy&#G3kx2JKHxc+paRSh^8x?e=2Y-FXTj`|HWFouNQ>Qo zUF?Z;&;yz#vQ&ympk46S3Ty!`k}YD)B!E0qC9c#3iNW7j<3yn=u!Jkj2`^Doq{p=2 zw;#bX;m4YcAIxES{1Zufjb_msSf^@a7`$16w@J z_SpgHAv)9n7(*$1fmhiXkl{>8FXs%GK{w&ckccA#P5^h~FQ5lq$Qjkc15gOT{0)m%yUD7>Hi{>$q8IgxNihrN*|PAZ-p;Pb zB%|zH41-5;if*B|{Dx2&FFQdu%ap~WSk}sVkPF*osB@17Wh1Mn(=s2e%2M#QeOXKT z@uf_pIBNGwM{~U=$9kL8lzfs$iVV*3}?6 zRKDOCUMp8!Z6le#{+~=IQYY(a5{q-SKP%V&oQfKCI4z~!+Lev!zg#8rdQ&&TbUY7_ zb+I#PZ}lHuQ+E?+T5+`b^+UQ(QcWSwH=D51tkY)mi@oNOjGJy)>nxfN&eCnePE9o~ zwD;!wJ?&2{<3RLGhUqRo$Jz7|2is^K$ouWNP38V@+2-?cRLPrpB&6Q%z*K$p=ld4bPR6m!#KumawwA#$V! z#1r-seqcidaSs%NQ&=00;YZ-f`p7wzL__R|iU4Um0EUqZ+oj^{lW0&U@D3L60i58J zsSVIUa#S8m0wJP`Cn+x+p(aoZ@>4nBMHYz`UWMG?2)Ga)da38ae-$voJHcK8iw6CJVvw3rOL zK|fh1`^=QBGDUa`7TIqsFbAZBOqeuNV;9&JnWMiEV~*$t zV@jMmJtc1FXEX4LZh}4R%>2NXULy&zf#%Q!{Z|F(!PMaQjp-P)VpCuQbz=|Q%XWbt zjX(Ly0)nr6RhOXG&+qN8S8mqpb@krYJ<{|#zdrHGYDT)ad``Q#oER6E&pR$I>NOX* vyxZq}pZ!8La;bdBTSQ%4Qr`cZ`Gw%ir0%uoEwf*A|F3U-njinn2S5FPf=A^^ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-456ef7ae-b9e5-4e2a-bc33-b6afade33eac-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-456ef7ae-b9e5-4e2a-bc33-b6afade33eac-0 new file mode 100644 index 0000000000000000000000000000000000000000..425c4a4d3b24a4d61c06ec3616b8a7555ecb31ed GIT binary patch literal 1593 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~mer0lmaMyI%)B_sX;D(*f`Ay85E-2*>|V`{Vj5iUf8Ght{u8>YHqx3`{LQhHMGdRm zE+}>G^!er=ygMxA=B!^{I&pSA@=v?-Pk+tQf}2Nv{T1OTC>BgX&$ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-9dbada10-b497-4ef7-bf69-a3bdb5959a21-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-9dbada10-b497-4ef7-bf69-a3bdb5959a21-0 new file mode 100644 index 0000000000000000000000000000000000000000..0157b8125f6806f67451e3858bf5324279e9026c GIT binary patch literal 1442 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~m$gNo+)kfr&iThaIX+j-zkK@7lI|Fnn7{H8|GwD! zUb+0A{hxL?*Q>0`n7`aAiWhUHxox}aa%|SFYpTUE(=M2*-F|fKnzpR0B)6-%i0whn Ya7I>!1mVKe35<_;5;|lQn9$t|0CM@_CIA2c literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-cbbe0867-55db-48b3-a617-01a8b5ea66e8-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-cbbe0867-55db-48b3-a617-01a8b5ea66e8-0 new file mode 100644 index 0000000000000000000000000000000000000000..faeddedfe5daf1ac40c8fcc1d904563d5f0b3e57 GIT binary patch literal 1496 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~mA9gZBTad?io+%l^~%X0)oxurU)dGWo9AclYd`wBm=iWBNChbj_}rWtFO5=B;M% zz~1UoP1voYep8clXG*5UoC%t}^NnbD{AHoVHy0%-F);}R7N}^*Dewp`KNzX-<3Jz> cLxS)QsfZN|cu(kVXzk@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8KEL8;7t`}g`tzu56J9^vpob?M07=?Dv`>}tYk%AE8+C}xoA71a`D>=vZ zQ1+*+c-UK(`bVK(_g23z`+xAH1Vd*mKTG-JJ6pvjDX~6Inzg_+mnB`rQ{t`HkrtIZ z$HTbxe&7~o;P$p#;MkCokf|Wp;5NZv!qdQrNsh$_I2#g|@bR%QvTqPLVBlfkA@H0t O;Q_1U69z$aj{yL%sLVV7 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-8175e923-c512-4513-9325-bbf4a875b2d9-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-8175e923-c512-4513-9325-bbf4a875b2d9-0 new file mode 100644 index 0000000000000000000000000000000000000000..8997bc8ff8a41b5b512eb66edc7500abc7302f48 GIT binary patch literal 2224 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8ybCo-J{Wr(e4N{3 zw5`;+fuoB>W2^pO29Kxg3{1T`dJUGDJhKn_Oh^k|^h0Fe=PibT*ZlJS`Edot&k9%FiY;;|35AX1sgUvJh{K)$7dU6qhl-QgUOmuO`sxrA-S{Qyyt#HDw7nLDR*Dk*BSYWmL!%?yX;YF+=tg0} zguxCQgCXqVMA+F0f(P*+GUm@k6vc}=M8Si2(ZfWUUy?S-nzh}*i?_Dl^FH7EKHu{` zjSgR}sXxG9Q0aP)mr~>eV44?LV*Rp)l_)$?)0Um!I6$4pb zN-LRwNtO~bqRQ){Q9dGxvY>&Ep@||B*AE$%jMDh9aF>+PLEi5G!XzCkf7(khh(ucm zhI6{8ZK3Fj1;Y@PCqYp}P@IKg%fq@!3fiYtwBfh{dYh4r6OJY#4jIsNRm!BcG;N|- za?pj6l@bgS9pHk|810(PgNcwd7lxPZ#(`I#2obLa!3-RW2;|7KBLO&r*Dl;f3!}S! zf{H0;V&V&NQ^x+U@nKM=$beoWRu zhf$eNl_&p&**-E zLV#9c=q4E!nJT+{RqU<0!~+b8AL$GxKfa#UdLPqprYqX zO`qnk%*;%+sT=2|)+cYC|6JY`XAhd)}v! zfn<%#%Z|ute>OhEr`YH96NoaN;gJ9$^YOIIAwEQ6Q?~Ef3>#N}Ov?#B3qKF>{tPRm z(vn{zOYjmevkI@44+^}nPgkU>BgulTZ-~#TgwHEf4F!aA2HP(S~6P=nY0zPZ$CXn`B5TS11;3< zx>1}4CUKbttD^`S$8|YOvVKGw7DWX5sbDWwmf3MAhM@2%)rp03MAI#ni}?_aA7qC` z1qrB?xmeC>My&~B9-Y#l-grxNCouYyM(O`Ah5B+)aGC~05W1NTw*%7jnB9fYVw*=t zgPkCZGMHuYZHzeR2Do`+X>5RAibS;~G|gbVREY_U^|L;8kOZh2EDVKNHnr#%gG%em z2kny(;;cyNd8r5~KC7^)Omj*QMFFOf%~zI$3jrKZfUz%g@v?$BORZ_408su=2!ynO z8JL)$*O10w(HFl`Cwk^YXfkM~j!~{ET&dWy8*mG8Q}!Z*p;la z+Gi^SXeEXYjEqv6$`)T0d#f&S4~fA?`bUGoREgzwX#bD#G{?ucoriXjI9BEyZzHR% zuambw1(G>s?ZH{Xx#Vn`ZTao!tgm&vayJ~9oD=F68?QNc9?GBFJ1LGKyG|^8fAnRK zr*3R@@y^ut$M2HTWb4Bl&9#5l^5DLUJM!#_*mg595&Lj)pr;$X=iS$~+~N-8f5gd? z&KE26?8;Nm=Y^R}or4+kMaN~Oy=3Y5AHU0CYyU#s0-*Lq=b3Jp{o4<5n$+Oil l`nC4g@%gLE(~Spbmzp`($C_I;eVN^k(WW;ITP~Nj{}=SZ!yf@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8BIY!0T=V2)b+Sub zv+ySlMxK4_8e8@MGI(s|V_0_0&nm!s3g3i%v(5-O`}Iq0Fnu59l5u*S-NAqNr~O-Y zuO=|Y)vFX+-t0Yj?s$ivrt{l>{}N=RPtIksVX$RzW_bSDlEHk#cBi)~j87i! zZ=65V{=wUN#;|}DA}5uzyu2#%=V+9!-zIvgZ-wT!$s5bogw{;m>#Kc5?t%3U*5iNX z-;@}Ew?n9@%uAWY&voI4fwWj{oPyWlZ!UTa3?1I61Vq=QNm2xHgNK zsW+}CXRd&k?8%i`*}FErWm()NntJl*y2tOt3^+J`*RyPQ68)q8f!%Ax#ehC7l}yK< zGRzLjf}UT65)3?Iq>?!t1X5buqLc%egv}4Kq$X};{&!xJ1u6O!i7 z-gDyC?k#(M3gwl}v`(+Hys+}fEt^eWKWblSPJcE>lwHbz!Ja#NRpO={1}n8aIk_@< zGA>_GxM_1;;1PFDg>KQo4cZTc{x|4rvZaWaWK2EsGH!C1>aOA`SLaP*M2~R*b75P0 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-0 new file mode 100644 index 0000000000000000000000000000000000000000..cbe9bbad3ade17fa40c65818091e73b116a67f1a GIT binary patch literal 1184 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhB=QtS20-BawmJ7si^Y)Cc(CkNn@-2Uj~n@EDWDG6%;n8zLa4LZ(pk@KA}PMUUc4k z&u1z5O5dloPn))H&qFn%j+RXlPLUTR7=M_?EYxg4?+bU79mC9AN4Q8%Lqx{CqT50y6n literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-1c83fb82-6a63-49ff-a80f-71ba0f7233b2-1 new file mode 100644 index 0000000000000000000000000000000000000000..4277b15b451abe8042e287d19dead1e2b6a9c4d3 GIT binary patch literal 1117 zcmbVLPjAyO98Ig95keE4btV zV3g!61Wl0PbZuQsFp--%8@I4{KcTG&j?+AA&5Dg_Mj7T*(xaFrVF9|wXTK|^WO(hUO|NY-?h^)5sj(1PA2 zu#feDXL`Z{ay{K!R{ckK2(1$@h{rY^G}JA>dfi~kY4L}9TR*x4%JD4=_4>Y3x#cm) zPLl+Pyyx*_#_&``6LXr%PR-B3ETd65YJ!kP_%z|5E1Bh}tTxN-9Y9-0rtz0>rKNqp z_Xv6m&_YEve@&wDF-W5ubA_%2htS$gc2lZQ9rO>Yku2Q)%5*i+6}$gO9+g|OzmDV~ zCD&dR9A_bBq4M#`_}=Z;hv(1ke4k#u`Sqy&sdo3n{U6G~+q&|;(RiNXERJZwRZY_( zE!y5ycLI&5ZEdfk1|fc+w%a?D5V{wTPDkDTcB`)35TmY$M~ruGYI4fe=P$d;)~ei3 DJa1x^ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-0 new file mode 100644 index 0000000000000000000000000000000000000000..0025b4cfb221a47fdf673f45b5562f8683e82004 GIT binary patch literal 1148 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhQ|!+Ez4HuI2CSVJ@>Sw`WVYMMvblde;GWcGBZ4KlaSw_F6%K#MUf}8!$Iqf?)2)2 z&b{UntG(BUzBpSpQH{Br;pid}M&|WVm4Qlgj{ON;oL6+-wEXUhq;8|TvgOAW+haIQ zK6K6Y7LK0v;kt_bOFQ?LA2~Np?e$lP(3at7bL3=Y;963nxNXj9g-6|tBIvFG09=fK Ae*gdg literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-619f76ac-a10a-449b-a384-6ad6d69d1912-1 new file mode 100644 index 0000000000000000000000000000000000000000..8646a801e468ff790522c4dade9bf90add8bd578 GIT binary patch literal 1117 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhBw!FlizYI`XDY7`TTX=DnF(jj2c_@|1x;YWnx&u!V;I8n3tKBT3n)=Y+#X+W^S6Q zYXXdCT@xTjH_6!4P&dgmE!EuEz{1?b%v{%Cogx#11Q0SY01*%_n8OIdAl`uo#tb6p G)&T$z1!aW* literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-0 new file mode 100644 index 0000000000000000000000000000000000000000..7d34ea31011525ac0b3cd7e75b80f43c12df72b4 GIT binary patch literal 1006 zcmbVL&q~8E99Hn`VTeZ$0dJ!oyzLg(h{d#(wVmQ9A#1a((xxNLggFYnf`YH&-pOb1 zU^C}h*WsVtn&$hH@B1Zo|895hklu?_%YqS0=-d7b$my7Y77#2*MF^UJ)13XAUasNm5^Ml2!3(}AUjfYH99#RqRoE|^D&W{@5``mwa IUQQlA059r8&Hw-a literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-61aa9e49-d705-426e-adc6-4929d7fa79b5-1 new file mode 100644 index 0000000000000000000000000000000000000000..53c972baf9dba8bf3fabdf3df7ff4b0c65066bd6 GIT binary patch literal 1113 zcmbVLPjAyO9L=g;IE(|DCL~U?IB;l6+N~R^#33}E7LlZ=NjkKtvz*xNg7T+wyh)|X z2jJUq;m!q#GoJu)F5B_!u8N=NG`*E}ns;%o)u_%_lF0~$!ybeHvRMHW3TNdhceW!HF zbC6!eF%Wsromn&^t%|055Io!P4k zGeW~h?{kh*f*Da>KCnN0^Q@=J?)KN(lj)br$E|0dp8rx#ii$$2)r$nD(U9g`ZIPf! zv|&Tlv?EPDAmmUD8v89ZIBbUWMKf%)w5F>4*sCg!#i%G^-VC|={C!{9S(W+?Yg=1# literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-0 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-0 new file mode 100644 index 0000000000000000000000000000000000000000..a505feadaf16bf0014276898aaee021592ba8474 GIT binary patch literal 1218 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhTzt~`4t;u(;mIe@)5jrbumX1v&L5azYHEnSs51js4Fi}e>p!s^y};B)&Cogs_nT^ zo$v7f!#myWccPhW9)4$XlTb7&cWpB0Vqx6SkRDb$BS@!rr_$>4>N8hg`PrnqbxP1g zm2(yf4}MCxMae-2o^KIVMZ~1v5q%|BkSo$~_I8r1JzhUX) WJN!_j;T8egXglZ>Q@3 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-77c3cc85-9a12-4773-aedd-0de9beb4230a-1 new file mode 100644 index 0000000000000000000000000000000000000000..7efc617f1f1dd57fd6bc4a203431a8bf41428405 GIT binary patch literal 1114 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhIpa*9Vdg7SEnr5Xt;W&_Y$T}j2c_@|1x;AGBGS_5tN(|$^K(Secr-Psu3Dx<~Qn+ z9xhodlhd7}`G02bypu1!{(5~`xS`4QZSt8Mb-zi|HnhsQL*Kkl6YJ{eQc z28@OoBcKLSoUDB3V;st#85`BHSRd2+7)ME#)~CfrG^Gr4D(SnO2l?P}CNH=cWabquX!+$vKPC(JIO{UzJdCwuEtf8|j7t4J2#2i+VRA4QN5{ z7T6bh&oez?0lA*;Evo(_JcL$>H^gI`4(jWcUp{Uy;WYoly{*5z1ojpeQ%X?d(HaN2c+YaHXYZ ze&+ysbI@ExHh)E;bQz@4wYfsqf_-SMC%Z0HsP?*tgxoj(U9g`ZIPf! zv|&Tlv?rRnOURxYGC29?xh!=BD1u^y literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-1 b/test/test_data/orc/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-7ae0c232-87bb-4974-b8b1-7a5d9dec3e21-1 new file mode 100644 index 0000000000000000000000000000000000000000..625a0f54a71421d608365815a1b0012fd28e9bb2 GIT binary patch literal 1115 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zh8G{m&Y9XSf27&LnAW&todw&Dde)pd%hwTdQv*i5#b4oYz0(4jh2 z53ZSNzg%0wqiYPi!3-y=CDRq*lZXsSLe^wU4&+QOBGxI1q4Kh${+sQ>@~ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-1 new file mode 100644 index 0000000000000000000000000000000000000000..aab0e6e316bc56ea9bfe338e4e36842224d9135c GIT binary patch literal 247 zcmZQ!00Itnb1{7lWiDA3MlerKgVs#)k0b)lW_6Fi`AWjA1LLlY>VnzlP1s`>=c4mgvj38qe7!sQ& W=7AY##0DrIWH>Sa`Oh=NDF6VW4-RSo literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-2 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-6676ad25-8fe9-4fd1-bdcc-a3bf3599e5bb-2 new file mode 100644 index 0000000000000000000000000000000000000000..c97470bd42b09b11737d1fd6953d36b54bfa8458 GIT binary patch literal 235 zcmZ9HIS#@w5JheLZ6^W37Q()See3CwDCj_<;{cR!Aw&V^qU0R>SX3G5jou%PEe!+< zYMqZEory*PIhr+9IWtKSib12-4x50}+-Saprh2xfR;_WV55r`%#iTrdr9(v>51yO) zdQqvvr)zYl!H^;X5|M;#$d>HMiCoB)q&reP?l0vYEAf=+CqjHb@y$u5z5hAP%+tFL H=iTuK|4Vs#)k0b)lW_6Fi`AWjA1LLlY>VnzlP1s`>=c4mgvj38qe7!sQ& W=7AY##0DrIWH>Sa`Oh=NDF6VW4-RSo literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-986b5fda-9721-4e64-9a1a-36b9bf2ad357-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-986b5fda-9721-4e64-9a1a-36b9bf2ad357-0 new file mode 100644 index 0000000000000000000000000000000000000000..484c9f7b79484e47c9105c7286afda11061b3c4c GIT binary patch literal 88 zcmZRHU|?WiVr1X|QVa}RcTAkh0v2K5g^0Yq`Nje)g+>TL`MhZ2j6l9;h*JOnqFe{1 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-0 new file mode 100644 index 0000000000000000000000000000000000000000..a221074bd3db27a936c7ef1d9179dbd9b8cddefd GIT binary patch literal 235 zcmZQ!00I_fHgO&nMlef)ftQ5|$YN(>WLH<>6J}usa#&d;`K9GpSinpTEhc3aRv?pu zU7J}zL`q(Tg$>AKV^t93P-S6Z2eMh1Ib}pyIKZM}teQexESx|#D~lqxnkPvf#KObv3xKCjaUNZ^P!1@ K{N)+q6aWA^whizA literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-1 new file mode 100644 index 0000000000000000000000000000000000000000..aab0e6e316bc56ea9bfe338e4e36842224d9135c GIT binary patch literal 247 zcmZQ!00Itnb1{7lWiDA3MlerKgVs#)k0b)lW_6Fi`AWjA1LLlY>VnzlP1s`>=c4mgvj38qe7!sQ& W=7AY##0DrIWH>Sa`Oh=NDF6VW4-RSo literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-2 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-2 new file mode 100644 index 0000000000000000000000000000000000000000..a221074bd3db27a936c7ef1d9179dbd9b8cddefd GIT binary patch literal 235 zcmZQ!00I_fHgO&nMlef)ftQ5|$YN(>WLH<>6J}usa#&d;`K9GpSinpTEhc3aRv?pu zU7J}zL`q(Tg$>AKV^t93P-S6Z2eMh1Ib}pyIKZM}teQexESx|#D~lqxnkPvf#KObv3xKCjaUNZ^P!1@ K{N)+q6aWA^whizA literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-3 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-b441165c-929b-4f52-b27c-a9598db52b36-3 new file mode 100644 index 0000000000000000000000000000000000000000..aab0e6e316bc56ea9bfe338e4e36842224d9135c GIT binary patch literal 247 zcmZQ!00Itnb1{7lWiDA3MlerKgVs#)k0b)lW_6Fi`AWjA1LLlY>VnzlP1s`>=c4mgvj38qe7!sQ& W=7AY##0DrIWH>Sa`Oh=NDF6VW4-RSo literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e889a0de-d6c3-479e-967f-2e62654d3790-0 new file mode 100644 index 0000000000000000000000000000000000000000..26cce0f82e71f29bf97d77644006c18554bf2653 GIT binary patch literal 31 hcmZQ%U|@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~m(bBrQ_z$ zj*mplEaf;RuxM=6|I6U;{jL_| zweyQ_>SVb;PxoH?=EtbXpw6Jh@GRef<%IL5!r7{EVcP}1JyfeYdt$qorTfJ>C@_dMU6xPVI5-_N`sdL!39ZO#jc!TK_X%?#$``?0@`i924e6 z*xcZ2JwBt^CTN?MeDC+FQmZ4=?p-`Ko%`#794;;vhMHeV1`0CMBLqY5NU9$^x?=W< zOWx1FWM)@YhwQSOv*B#X8s8%o3w;H4w;0%6iQ=#PX%sEQ!19TE4toL9o`!<`=dM*Y jpKwrHu$$#L_nW4vZ7;Hx8L>SOIAD>mh35z(de8s>R8Aa1 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-55558948-e419-4c96-912a-f7d4b1d25af6-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-55558948-e419-4c96-912a-f7d4b1d25af6-0 new file mode 100644 index 0000000000000000000000000000000000000000..cb453879d7bb8d04b0878b150a8777d582f016ce GIT binary patch literal 1644 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~m~U_U_#tc8|F^7}FUHZdQta^$oU_{R+%oN>X=i5ebU4_YSrU@BEOqK# z|M}gkv!4bn3)4SwDu6~&Wv1l@2JbC z-0|hF@s(U2dhB#&(V2*4OrApQ?I#R3EU%s{n{vkF=uIojb?eF_ZYK)5u}m~nm1kgn z&RE!BJc087HS)b)VTLI;!Pe=9kb}=i^u)m%?tr=5WC4x(|zjUXpe6stJ!B Z+6*T1rm)+oHE?}w&2z|;>|jO@c>qN`I(Gm7 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-cb21ca4f-0678-48b5-b2ba-f797e7dedab4-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-cb21ca4f-0678-48b5-b2ba-f797e7dedab4-0 new file mode 100644 index 0000000000000000000000000000000000000000..665086be6103f8ebda049de2d0027b61c9fca873 GIT binary patch literal 1704 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~mcu`?U1;j@@b==p980H6dT{5% zrR|=ItQQ{#)tty)@3;4Gx|A_nJfmGFCoc!ToPvmfl8Qw|0>i>H7KOW`SM&$nF<0K_{c6tLwhJF$FtpC;kXh_oczfR5%kdoFUz&WglZg*X z^Gcg`DPT_Wp@T8Jm+z$tS0CwI&n0NWJ2yy5Ns5c7!{GUh0)zBDS<>6Je4n3q(_Om$ z)Qgo1y}QrdUTn+aX;j?M=rd{N5hu?{GdUHE#k-kQ#f2h`+Dx{+QA&PYva9J<{O;eE zeg2p*6-}Pb=8Jjlio~PcRX+Dj(an!{;*mr zGc+{Ru(F&|ILADX!KmSv&>Z$-j7J>A*jyMhAMm#PF5Pxc+J()C@vlPn4{oN$*oc?2 nIvI}0=B9g#TAmYv literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-1b596e7e-4a38-4963-8e62-1c50fb11c73e-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-1b596e7e-4a38-4963-8e62-1c50fb11c73e-0 new file mode 100644 index 0000000000000000000000000000000000000000..2284d566876936ef5abe6fe5859b3d8f46c58c77 GIT binary patch literal 2592 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8Ua>4{njUeLYfqTn zyd}%EEhS!YYHZd2%ixhJ%%HX;+cY8j5?g>>OTe1O>dXHXx%o~q1Zd2-z@zdle&yoH zGn1q`9el*$hWEhPbN@e7ig+F(oj>Gej}Wjc_en!@F?#pYy9GWd90q55LWG zWQI?1f+I7(gvE>y1(y>ZDwmgdPG)OJd=USzW>G?8%CkwCJ)8;&%u8ly6kIXQlZZ}! zp&7g1U(#5+D)(yt|E3F9LoLi1JSTgl2^mb-J^$a1l%AU>1WvjJPC3!Qxj}hltYy(V5xN_fGxWxGZcfqm0z66(@Rbt_&?d`!easuT_ydWph>>?OVLIa^vgm*;C8j zEL$_Zbo=C0-vbJrjupfx@UaVVDo*rJQPTR6_`2R&E$!Di+uFBly5e6(Jr#*rcbIkS zy}d2pzPRyLu23vB@8UkyFT*@xvWh3WP=i56iJD@9K#!3}z{~{_jEPQN8Z#y@Il%lh zdz-U-tm-MP$!n_3KWc}p*Ky+J@yt~bnw7qL;o~)IiVN9;8<+1|koHj8=9v1D9ZgcR z-Ph%>xmMC$DnET{viT&5&ud}~uQFxof8w0XnBp%O6tsFlOtT{AQOBw%xAMcq^JjNx zG#y#@vAr^HqV&AEHX%XJT2yk?a!t4sxpLcVZqAwI(YaDm!7cSjrrws{Ul`HTApqQz BplAR9 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-24630e00-34bd-45e8-9ffc-e1b061217359-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-24630e00-34bd-45e8-9ffc-e1b061217359-0 new file mode 100644 index 0000000000000000000000000000000000000000..ee08012446e1ddf2f72ffe12826aa18c5fc3d0a4 GIT binary patch literal 2251 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8en)@VRr@XI)Ul#( z^FPHL?BY1UqOn!~FN4Qk4u+uf2AUIA&S19R_noug?j~E<2McKl|Plx3_aH>&9eCv8r<|wOF9}(_!nf$=RyT zaivjjBXhzGr+nV{&1m6&!ADAZkCZMg=tz?~>eKyZZSKwd)Xle!WxR9zP`Z0X)`klK zf{F|ObN+k%wTd-Blb=bB=fARmXUd=QKdXLe^gqgQaNtZikTco1)c(q(hqh<;_MTYL zTf8&%h03v%+kP)14(!-ae5g+ zuv_8c%P4C7&IU;5C!u{bMldN9l^<4z6Hq@bq7B2O(JPFso-kg_V~~NkBJ)yYS<@Pd z!jGCzLNp6Q`dS#L&+j$OmV@!&UZw#a(HjT3`gpL`Vi3f@P&7sx@~lb#M&;EDSI|Q3 zt{Efz&>+s!U~!b!PY@|N46-&LbqNAMeoEVmm1VXYiNPzJs5&uy4m8~YiGU4+xpuZo zP=G_NOau~UGipuf^Q0(s>Xo-7qKt$dukzRcr4e2$YQHU zQ%*PZU<__r{2WFCHbcU=F*`P3D@|hB5*lXEUnk&=V4Rb4twdsn%(1je_Py%GE42WwYvZIp^0BsWl0(td3rXuFxJ;pJ~s6A z)q(w;0L&+^pV;WBF01UVdC~TH^bIYnJwJ!pc9;h*!a#(9$EgPoA}WG^5Rv(kv`N;i?G9eNwSC|3d%xfJzTel# z)U}4D(3{qks@xKnj^GA4iWu?iV|5hN%)A2sJOo8b7DgOF{`HiBK|y${5erl zvKhZYmPQ$*iW<_(QxcMsd7^(}uFNEi0FI|4EIusVC1o_C=(7PsG#jjbI!tnqhPw!+ z3L47qq6kE(AponBzz8H6!9?-ZVS_XStkXK$2to;B`jL2)Mlu7Su+CcI2 zxC13;D==JSRG=bJ)-jt46Q)^V2#i^c6K;LNWUL+pGYC8+p)Jq81Q2vyt8fo3^zKGT zCJJogDh<|0u~CLG<*>;nh|G*E6WC8Bdx@$l7O)tSh7{e2rE|j2EnbZKh$KphGqOhX z>y^cL(P>7b32Ppc(xTpbOH3yS=9E_H|1X99ay+;a1uTW#91RBmXM4=;LTs`9qj@R> z!lXbu7T?9lfDyphH&(_5h(V4tme4kX^->iku-4D{)D#CuJy--tJ2tiK7mLd1%ZKfg zmJ*^2&Aikk1?4qS$@M6bEK4|zT)uK7To5n-0>Qn^<;zOuthA50Qm3Xn$_@(Q}53`ni4Bd*;df<}GUc**E1t8|XJpnZP{@zl$YfEfyZ+X=8!r%Mu^78dVYn3hj E1)cM@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8bU6>IoH_a7f&|+# zy^RkKuMzmcrmt{^5W2EYNh{^t%v7DmkG8YCX20I4#2k$<7_#nXWPoCpILP0q{pFmDwId6H#2?d@@ z2NV(=EqF9$?pwLr%9DG!=h{hgH=UGvWz%=@@sBC<4n#H{eK>Kqa6`AxK@p#i2ND{m zhJDlMl;v8we|63CvqzRMwe;R;?tJ7Ji(Dnc!7ZDA>14g0ozrvb?|qkJmQ!CIn_#4O z;Xo6cxLUx0-I?F4OXupk&o>bf5GuU2IkG#-{?zlMJCB+3KALp;^=zw4f}HH!nMr$v zvN~1e*BI};Wz%aF`oHF5v*?F+0xZIJs@H74y$(Y|5P3 zk->!@L-c2_-FmKd%hXsQUMHn38&=)byRUIIXin!^UKfRi<|dW1jGGc3aLXw$ME>HB zYj89Wc)-Ziv-J7|1|GennL*3E-;_>2=&~lkOHw8F*!=usJ05OY;2~x4FYUU3zv6l| z>DGNFi(8E?>dMZq;GDGn;l~%<0yA#%MjJ`V>BUT&9{a;y-1CM1fh$}KdNwVNT*}J- Q;7!ou13kv4Ska>s06i&9uK)l5 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-0 new file mode 100644 index 0000000000000000000000000000000000000000..3772c770f51f48f13a1441d4e94d672df9c487d2 GIT binary patch literal 1006 zcmbVLO;5rw7?$XngE9Vqp2@s<*u_*<<6^fN_Ms7%rfdaf_CdO0f@FV2{~Z_qfCpDZ zHW^Xmwyw`dpXY6@!Bgw#ls@xJ$%8RV=%PIbVm4tw1)K#L4?zc{G-<5!2@T}WjE@vr zevg?lp;3~h%2I5^QpTxZl0J-B9Ogjn&%c$3hkCYdndRonD@p|@Iu+nb$B3BFKoukx zJW5ARl&wG1P;s__FlUnVAWk-JD zcR$G2;ZZ^6R_kU!+I9a~BAPZS={E`crhLnc~i?f#6HZR^^&kwbaN9FAk DCayx! literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-1 new file mode 100644 index 0000000000000000000000000000000000000000..0265b6ceabfa322a2e692ea33807ad6e6eed4a08 GIT binary patch literal 1118 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zh9wV9M(6H3-2PVVXGIm8={%-gj2c_@|1x+iWny^LA}G1QlKnrk-m>RY^Vknq-pHFgdU$xl0MC{Y3i!grwyn2Z4ONA VJIcVs!k{2LhrOZEoH+p9J^(#Sc02$8 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-0 new file mode 100644 index 0000000000000000000000000000000000000000..bf901448da3b69c84e7779ef3633cef130c38443 GIT binary patch literal 1196 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhOl$)Gy2+-6SCSmFF(k1Yhb&^q_I{1FN23DE5oK!ii!)=U#5rt{qrOK81H$}=t*TC z_wTHJ@B3uhoR!~LZQN(v(JXURIoImQIA6%~Ow!3Ik@u&)dAZ?B=QF#&Y~N_Dj+R6g zod>~Dogz=wbmkvlv3l~alovlNvmTz>TD#AB{@dNw;^McN%ov0|a2dWh@mRt7Tl=*u zmz93Cth9Wz$a0pu#SOQF4P0zpoD2>OOmkZK+Jzrx6m>A{X!^`x@F9(X3*GeqvKgO( literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-1 new file mode 100644 index 0000000000000000000000000000000000000000..498dddf1189bda81673ad68a4ac21ead60e16bee GIT binary patch literal 1122 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z2EE*!&lY?-6YHKoLGUT{U3DyTd0FE})?>f-z5TzhJNJ-0o7zoB*XquL2Q!0(Jp*$-+X8eu0kH#jYybcN literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-0 new file mode 100644 index 0000000000000000000000000000000000000000..e8696c2d1bc4cc93f43d17285c3d96302ace879a GIT binary patch literal 1234 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z25OFR)>w8p;(vD8rk>q)Y(dNO_E6%zu4`Y7#wVvz!a^l^*P|wvq zMlKG_|5O(7EcZF_p*r2Et#8?XwbvR{>O4or`k=er^UQhx2~jn^}Sbe zJ7>0e^X|>Yn|E7luQ4!YGG-C`z+rfD;_)P_U#Fs8EEnACmDP76BkVI5=UyIx9}5@+ nc^DitnTl>EN(nSEq%S=D(8Xewj1t4T2Q~~mdn6l}(ESJi8vv>{ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-1 new file mode 100644 index 0000000000000000000000000000000000000000..c9a297ea3c6d49fb68c05d4cb9db6c7d03ef0d15 GIT binary patch literal 1126 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z1}~#pZAa+~c}>$r-)aAnU%_;WQDdwAUj~n-Obkk`!m|yJfxG((M zckkXE-|+o+@z$9sTxM!r$&-G(zx}U%@x?MuhEs>0%qv}CcVvlk@PcVi19?@g`JS!| d)wT6Kk|@}8P?XJ$g+W1h4tqnRIdcHIy#N~=c_;t? literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0 new file mode 100644 index 0000000000000000000000000000000000000000..292cd2e714089487f7b7a2f9a33c38fb10acf1d4 GIT binary patch literal 1118 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z25+9{*|K@2$sUR>@6I!7H8AaB)Yz*3m%(Ex6T_nxLCFP{?EjhdmOY=E$9}-_c1EV> zQJz(Q!h}2-x4r+r@9lqcqfNFJ0&-`si%kqE;<~IR^!Rj=^l6q&Q&+7%Z8+6$b6{fM VQ3fU!1_j|c>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhG3 zzR2vP&t9p536sjWzHxHA`@iq)|L-3+`YM?42pMhLdF535jw2z96EBC%$P-<0%PiN9 Z#Y8`FMl7Eq3xk629QKArbLIeaO93i$a5?}0 literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-0 new file mode 100644 index 0000000000000000000000000000000000000000..be199713eaf4c90424fe6066afaf3b5a66ce48c2 GIT binary patch literal 1159 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhMkjN8du2OOgwsYciNHcqZ3(rm^8NP|7Gxa%gk`eT}pnza?Ahr8teY2-uW=+#-;l| z@AsKayr*JWY}cTF%X1P_eWXDTr-{k+tksil`Q3SDlytHpcwMmi%*e~z{>t}v9RJ_Z zp|Ag+)tsSK&3S@Vb;RA5DtnF>nNBh)@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z27N&T@vM+1iS0}7nJ!UwD_}apsIgW5FN4QfCI+RWLNXhcTJmt`d9o?E2sod(#jM`H z<(TgiS3`w^Y@W-$ZTyz}c3=MQ|Fw3vTe%Aj1fDMJS?1Fj7&DtQ(1Cx|WNzV`cQ|-1Q;Y3u!u7WFi9|xW(CwJd!W^_T&zG>0o^6S!KlQ?#gd(xS|9`_B{%{Y z*cll77*2u72%x(EAORT0|O@m14s{0$c@2a0|SG`R{g&U3u>7dgj~4!3|vp} zIea;@;#N~aQVOFyt0)T9qD)*&r^wooV?w0eeRA)zJr=udd}|(3iRMlws4F zC$nern6S@@5actG*5W?FKB@7Uk4C})sL}0vnHWB~2udDMmHq#J+5NmD%Rl{JF?m~l zCb#y|yVY-}Ffo>jExK#WbIbNH*F#eSrGPmSS2Dju2Ksl;2sd1GPQ7N=a;6#656oGg zs1Ni~7^lIZriJU+ekSxpvMDfuU7s+G!IT3SQIgN@-PA|NS<`-r14m zQw%%U&K!3*@-poId0!XPJsG?7Rz^sFezLjK!|N8~H13C|7V%vFu#nr6dD z*MrX8j z=HFwke;s6TD(#_G%reLPSij9FF{!H)BwbDKKRx*JhmkC&Ys1s%a^=Sp%RV2x$k4gw zS@*Teo!b^R?|9SG8L;7ou=hl96@$Cycw&$FJn6Ds`|P&(&rRyqo`Nci{Y4(gu9Hp> zNx15|>`!N}a&R3FgQnA3p@bGz=UCoP2YnVdylZs1<74oEhe3ps U5)%WDMzR5C(-&rD{~%`x0DE5W761SM literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0d30be46-8b96-4020-b2fc-ec814be128c2-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-0d30be46-8b96-4020-b2fc-ec814be128c2-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..2e0462494db9b28f278874511cf1d619cd26a260 GIT binary patch literal 736 zcmeYdau#G@;9?VE;b074&;~MvxtJLk7=(B@7=_q4LORepEM#5#h=a|5jX~jsK!C)lj0yY( zvrayfkYWORBVhuAEC<6PJ`1A&-#Z4z-e=C7*|KWYMah2~{pU{KyX})COJLcB6$}N- zTEaKI?bBZP`qDblwdy{?N)z`oOSd{i@*QJ&>im780)y9k<6|LO)7ZMc{b8K>bgIc^ zZ`ptkgmgs+N0ydR5wytH*yQ{M({BljXLN?&fvR zmd}lyJvDPiSfqfMd)MPlhCv1l0*vAHx1^@{T%FqI-PXSN=)_k%ZLiw+k}mxeUzl)F zc~+CYM@{T=9@+nD6|VcEWPY>G|Ke_cEmLbQKVL<=Ew2u1=sbn(buO1DMBDQ+Xij=? zNpXR}l#2@T2_?px8F#S@6&fFGFlJa`^J0o5#{?z;R*8lN0VO5|9*r0S&ZaNS%>F^n F5&-pW%FqA+ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-76bfec84-0851-4a14-8c8e-5e9f54c18ef2-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-76bfec84-0851-4a14-8c8e-5e9f54c18ef2-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..ab399d8239c05a8dd37e42b6ad916473d5a5b4de GIT binary patch literal 916 zcmb7@T}YEr7{{M;-p{?x`R;Y<%Um#r;A~U(rPERN`Z2TAxlC<_Iae+dF-;BfCZv+A zzzPX7il~Hvk%&0(LabDztiYhFknqAH>Nbcjf{vR6f&|^1^E~JG!2g^-|5IC6rUejK ziRP_-$RlNnKmmZH;jI$SV{rk=-wb}pAw(A^iG*-Gh>h~5#t#+(lH)*4080SKu**3X zu^jkM2}!<3`C&hiFvKNNc&jN%pxb)8dnCFu9PW`fupgWND;A71BvXl2tIf!CNRqj< zG})nbW`_@Cn^0DKlq^MYCgTCEW!z%)x5TF{7qS%K9^q$O^}7LgZ- zWeU}bk;2sjQYae(3OkFC0>A-OpmO5w)@`@$x(Z;9=KVHKe#Wr4oG2>c>)g`)n|;R0 z`C0Yid;2~|(Zs-&fw$jZMHfa=E+k|}JXdd~Pob8a&B%tXcxr0=#?wxe1bK$_(Xq8$ z(OJJR{#xl%)>j`G-}rwijx~eWehg8Y+d9KF6qr_0jni3sxSVsDD62XBTKSw#TdioB z(9rF}GmgyYr>;@mQ1mfZJThAO)0{e7ug*ANUOJ8v=e=#_>BcAT(wZ-3ho2?TNnb2u z?UPNN$uE~fU!Tw4$$z*vR8K+0bgTK%z>JD%!Li4+-h~_$Oo%)sexUSfvMnMxf{bU( zR?|UVEpZ4_G{bUwE?DJl3U;*V#o$rzkp{1?%p3GI)RcPb^a;VnlIjL;uIrppb)U`cXp;oH&Fyr%?1c^&fwaT!a=Qwh1+D_p zp?s4><~7J>caVy6+N_>f*gfLThzAUwq#e-?aGtGS@Hf%{CYIrNCcSCxIYA=fgSAbAIRWJm=5<)YZGv z00dTKc!L*;NSG#gWQk1Ppi32)aCcWkWUjQfMx=G*1uMX^9wDt*nUO+JfI^T1Eb7reZ#DsV^oZdFC1#W=ECOQKL@cNi zB1wsEyA$RGBA%0OrYI-Z2$YjH$-v3Zk`#agsG=%~&WXMgU9XT9=8AUmo^B%e&Zw}r zdV?|zk69d_Q;V_|WIMCwvJWRh4Y-WPD9Ib){*Ig!!qAz{!Jjt!rlvEFcf;|-1Q;Y3u!u7WFi9|xW(CwJd!W^_T&zG>0o^6S!KlQ?#gd(xS|9`_B{%{Y z*cll77*2u72%x(EAORT0|O@m14s{0$c@2a0|SG`R{g&U3u>7dgj~4!3|vp} zIea;@;#N~aQVOFyt0)T9qD)*&r^wooV?w0eeRA)zJr=udd}|(3iRMlws4F zC$nern6S@@5actG*5W?FKB@7Uk4C})sL}0vnHWB~2udDMmHq#J+5NmD%Rl{JF?m~l zCb#y|yVY-}Ffo>jExK#WbIbNH*F#eSrGPmSS2Dju2Ksl;2sd1GPQ7N=a;6#656oGg zs1Ni~7^lIZriJU+ekSxpvMDfuU7s+G!IT3SQIgN@-PA|NS<`-r14m zQw%%U&K!3*@-poId0!XPJsG?7Rz^sFezLjK!|N8~H13C|7V%vFu#nr6dD z*MrX8j z=HFwke;s6TD(#_G%reLPSij9FF{!H)BwbDKKRx*JhmkC&Ys1s%a^=Sp%RV2x$k4gw zS@*Teo!b^R?|9SG8L;7ou=hl96@$Cycw&$FJn6Ds`|P&(&rRyqo`Nci{Y4(gu9Hp> zNx15|>`!N}a&R3FgQnA3p@bGz=UCoP2YnVdylZs1<74oEhe3ps U5)%WDMzR5C(-&rD{~%`x0DE5W761SM literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-e32042c3-9e41-4cd7-8bcb-59e0944b3803-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-e32042c3-9e41-4cd7-8bcb-59e0944b3803-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..ab399d8239c05a8dd37e42b6ad916473d5a5b4de GIT binary patch literal 916 zcmb7@T}YEr7{{M;-p{?x`R;Y<%Um#r;A~U(rPERN`Z2TAxlC<_Iae+dF-;BfCZv+A zzzPX7il~Hvk%&0(LabDztiYhFknqAH>Nbcjf{vR6f&|^1^E~JG!2g^-|5IC6rUejK ziRP_-$RlNnKmmZH;jI$SV{rk=-wb}pAw(A^iG*-Gh>h~5#t#+(lH)*4080SKu**3X zu^jkM2}!<3`C&hiFvKNNc&jN%pxb)8dnCFu9PW`fupgWND;A71BvXl2tIf!CNRqj< zG})nbW`_@Cn^0DKlq^MYCgTCEW!z%)x5TF{7qS%K9^q$O^}7LgZ- zWeU}bk;2sjQYae(3OkFC0>A-OpmO5w)@`@$x(Z;9=KVHKe#Wr4oG2>c>)g`)n|;R0 z`C0Yid;2~|(Zs-&fw$jZMHfa=E+k|}JXdd~Pob8a&B%tXcxr0=#?wxe1bK$_(Xq8$ z(OJJR{#xl%)>j`G-}rwijx~eWehg8Y+d9KF6qr_0jni3sxSVsDD62XBTKSw#TdioB z(9rF}GmgyYr>;@mQ1mfZJThAO)0{e7ug*ANUOJ8v=e=#_>BcAT(wZ-3ho2?TNnb2u z?UPNN$uE~fU!Tw4$$z*vR8K+0bgTK%z>JD%!Li4+-h~_$Oo%)sexUSfvMnMxf{bU( zR?|UVEpZ4_G{bUwE?DJl3U;*V#o$rzkp{1?%p3GI)RcPb^a;VnlIjL;uIrppb)U`cXp;oH&Fyr%?1c^&fwaT!a=Qwh1+D_p zp?s4><~7J>caVy6+N_>f*gfLThzAUwq#e-?aGtGS@Hf%{CYIrNCW8%zksGXww#YT7Q)G|8=C;suKvG7uBD#Y7{9qJe7{M@2WJAtH8*U=cwg0z}y5 zKhEJ|;6FzQ;yU4mJTxMCYeYP4mtq)2b5}<|P@HOM35X#iI1KPoCTU5RXyJ7!79GzE zyu_ynLV-@=b$l$(s{leW;7WjOh+zC59`ypaWKu2QB+bxj0Z?a=Ppq0lSV1!^7j9}) z$w*bjc(D8Y)6^Wjaxd#nN&f~H^EnGGC- z5j7vsw{Pxyl&Y|O`845vAa78)mYk{n;r`XHE5T=%9%6Mvk1u!4RDSDR3Wn~FY^F{w zy?USB-PxO=22=9FTWlif&FiwhK{n8IHz&|$(1&WZ#OK-hbZl*V(5ROHiA>PW>i4jF zomUt3ze~y}&_tOICLbEOv%F#`Ocbunt8@p3g2tG#^8{Y&NoS`g^`_~?nlZYr@I_px z=A!N9)3diGf{jG%nmpFGUY+)QP_;b%`$yG>+?nADjDV8adc*DhIhC>wkE~Ze4jC$# z;%I{VPH0%v7BO)N)3zGy(h+1Xh2exkNm3fhU+S#(w>E1yf2Fg+>vR=4{VwnELZ?R) z<*zCz^E&;d$6Uo4JkX=Tkxbd$)X~+VA$l54N=Y=~NMKZfhSt*^?Z(z-fiY*wS+dn^ zw^$LWu$Zl~)t+Uu+Rzu4EeS}k#xl(ogs7}cyCZT64sPcG_#{Wnj^O(!NBBnm2HvNP zBz!BfzC1_5u2J=w@-MrNf)8s{nvFR|yK38`6J)_XZG(J`d=fqIamaN8qvvo=DGm&9 T5)Q24Os4AGPl71-6p4QTfqKl@ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-dc40f35c-9fff-40a3-a08b-38898a7538a9-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-dc40f35c-9fff-40a3-a08b-38898a7538a9-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..b49d67dc02be9e31f052082e9274ba3b7fccc4a4 GIT binary patch literal 946 zcmb7DZAcSw7=P|=ySts|ygL=oX|W8%zksGXww#YT7Q)G|8=C;suKvG7uBD#Y7{9qJe7{M@2WJAtH8*U=cwg0z}y5 zKhEJ|;6FzQ;yU4mJTxMCYeYP4mtq)2b5}<|P@HOM35X#iI1KPoCTU5RXyJ7!79GzE zyu_ynLV-@=b$l$(s{leW;7WjOh+zC59`ypaWKu2QB+bxj0Z?a=Ppq0lSV1!^7j9}) z$w*bjc(D8Y)6^Wjaxd#nN&f~H^EnGGC- z5j7vsw{Pxyl&Y|O`845vAa78)mYk{n;r`XHE5T=%9%6Mvk1u!4RDSDR3Wn~FY^F{w zy?USB-PxO=22=9FTWlif&FiwhK{n8IHz&|$(1&WZ#OK-hbZl*V(5ROHiA>PW>i4jF zomUt3ze~y}&_tOICLbEOv%F#`Ocbunt8@p3g2tG#^8{Y&NoS`g^`_~?nlZYr@I_px z=A!N9)3diGf{jG%nmpFGUY+)QP_;b%`$yG>+?nADjDV8adc*DhIhC>wkE~Ze4jC$# z;%I{VPH0%v7BO)N)3zGy(h+1Xh2exkNm3fhU+S#(w>E1yf2Fg+>vR=4{VwnELZ?R) z<*zCz^E&;d$6Uo4JkX=Tkxbd$)X~+VA$l54N=Y=~NMKZfhSt*^?Z(z-fiY*wS+dn^ zw^$LWu$Zl~)t+Uu+Rzu4EeS}k#xl(ogs7}cyCZT64sPcG_#{Wnj^O(!NBBnm2HvNP zBz!BfzC1_5u2J=w@-MrNf)8s{nvFR|yK38`6J)_XZG(J`d=fqIamaN8qvvo=DGm&9 T5)Q24Os4AGPl71-6p4QTfqKl@ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-f818ca94-495e-4146-a8ad-35a4474f5058-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-f818ca94-495e-4146-a8ad-35a4474f5058-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..b49d67dc02be9e31f052082e9274ba3b7fccc4a4 GIT binary patch literal 946 zcmb7DZAcSw7=P|=ySts|ygL=oX|W8%zksGXww#YT7Q)G|8=C;suKvG7uBD#Y7{9qJe7{M@2WJAtH8*U=cwg0z}y5 zKhEJ|;6FzQ;yU4mJTxMCYeYP4mtq)2b5}<|P@HOM35X#iI1KPoCTU5RXyJ7!79GzE zyu_ynLV-@=b$l$(s{leW;7WjOh+zC59`ypaWKu2QB+bxj0Z?a=Ppq0lSV1!^7j9}) z$w*bjc(D8Y)6^Wjaxd#nN&f~H^EnGGC- z5j7vsw{Pxyl&Y|O`845vAa78)mYk{n;r`XHE5T=%9%6Mvk1u!4RDSDR3Wn~FY^F{w zy?USB-PxO=22=9FTWlif&FiwhK{n8IHz&|$(1&WZ#OK-hbZl*V(5ROHiA>PW>i4jF zomUt3ze~y}&_tOICLbEOv%F#`Ocbunt8@p3g2tG#^8{Y&NoS`g^`_~?nlZYr@I_px z=A!N9)3diGf{jG%nmpFGUY+)QP_;b%`$yG>+?nADjDV8adc*DhIhC>wkE~Ze4jC$# z;%I{VPH0%v7BO)N)3zGy(h+1Xh2exkNm3fhU+S#(w>E1yf2Fg+>vR=4{VwnELZ?R) z<*zCz^E&;d$6Uo4JkX=Tkxbd$)X~+VA$l54N=Y=~NMKZfhSt*^?Z(z-fiY*wS+dn^ zw^$LWu$Zl~)t+Uu+Rzu4EeS}k#xl(ogs7}cyCZT64sPcG_#{Wnj^O(!NBBnm2HvNP zBz!BfzC1_5u2J=w@-MrNf)8s{nvFR|yK38`6J)_XZG(J`d=fqIamaN8qvvo=DGm&9 T5)Q24Os4AGPl71-6p4QTfqKl@ literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-3940b910-9cc6-44bd-8a34-ee00e35f4977-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-3940b910-9cc6-44bd-8a34-ee00e35f4977-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..fb4c9d71292472acd2da1c56b132398ba0c3d89e GIT binary patch literal 755 zcmeYdau#G@;9?VE;b074&;~MvxtJLk7=(B@7=_q4Lpi z5=&B3gkTH_z5oVx1_nO{5il7ARQ(?$;Kty9B)oy2fdQ%$ELX6Ofk9)d{$GU!rA!P$ zF5G+puA7-RD3wY3cuZF6Ofu;=(VuaB^2N1QrN?46hzoCLQa@nr{6Tzm^ugw*tF)!?3p|!>~k0%9h9BGpTf4OVb8T`s^_34cYS4I_{1V4wZJs@fB&@(yF$ZC z&!_+S{o_7$d=b9w#^GVqwT4%IDvJQC0ME|j7k@m`|LgQfru*?_;R#zeu2l7##kDCe zE>`Dx#-(2sKONcF8FWIV_gb~^wi!u1VeZM>WXYz-1olJ1Oom(zh9rM0BZu5OhtF7) zNgUo_*!Sq=gUhP=QjFQ&ECpWylQs-=EnU~)$gRNE)dbX{33ehWi3hfn;H>jk%fEAOr(_4 zglDqs`WbfaK!Mu1f(!#I1)Vmp>f zw{FYf^cHk^{&*7WkG40giU-SNmz_!pI9Dm=z3(30;JkMwk`VMOR69VINj^b<;)AtBXL8pqux79)8cu|9PMH|JKyH zl>n@irx=qDa&VZ&k^sOf7!%Jhh+8CtFB%`1F-c93WK3uVxCUWU;R7QEssxZ@AjJR# z*y8}jQNld&pBlVyi}XP;*3cw0(ioF2g{9hhB3(Sy5e|0=8^{L^fTu=;vUpvoQmM?+ z+j-tlR;IEm9r_j-z>*P=1RxuBV(OpeCjh)gL@}(K14^#octDgCClssT07+S?g2yQy z*UrK*ES{N2HX}l$>wyqSs{}-8oFGI18lZwG$K&1ZcDvU#0K-DgFYUb#2$r8Db7FX$ zUb#KltF3wzmoLAu71;~#42}-I_`15VIFUAxlpS@AO=iT1Ci7-=Bef(Jo4#_tgGhlK z&HDA5uPvOTesTJl*ekBD&1j$dex5o&z9hQ))2tu==v)qvV6RBenm?FHo4bovXNM1y z@{85V$5g5xKi^%wapY>}wQnk5j z1&%yt9?pnd9gpi3gw_!pI9Dm=z3(30;JkMwk`VMOR69VINj^b<;)AtBXL8pqux79)8cu|9PMH|JKyH zl>n@irx=qDa&VZ&k^sOf7!%Jhh+8CtFB%`1F-c93WK3uVxCUWU;R7QEssxZ@AjJR# z*y8}jQNld&pBlVyi}XP;*3cw0(ioF2g{9hhB3(Sy5e|0=8^{L^fTu=;vUpvoQmM?+ z+j-tlR;IEm9r_j-z>*P=1RxuBV(OpeCjh)gL@}(K14^#octDgCClssT07+S?g2yQy z*UrK*ES{N2HX}l$>wyqSs{}-8oFGI18lZwG$K&1ZcDvU#0K-DgFYUb#2$r8Db7FX$ zUb#KltF3wzmoLAu71;~#42}-I_`15VIFUAxlpS@AO=iT1Ci7-=Bef(Jo4#_tgGhlK z&HDA5uPvOTesTJl*ekBD&1j$dex5o&z9hQ))2tu==v)qvV6RBenm?FHo4bovXNM1y z@{85V$5g5xKi^%wapY>}wQnk5j z1&%yt9?pnd9gpi3gw|-1Q;Y3u!yq=ut>0wW(CwJd!W^_T&zG>0o^6S!KlQ?#gd(xS|9`_B{%{Y z*cll77*2u72%x(EAOSZ92e_~U0|O@m14s`ROc^i`gJ9yq#(JIRSe{@zv1>o3F0rW6+nm^ORxJnkTbo z@|dvCi4f#7lGfrr!9J<+nU6-oKB&>{TbUR>u?R{WP?i1vf7$)KBg;SiUom-GekQl} z(!1TuMHoxPX5Do<@`|{1%_@s)}mb)5cl)@4@-MA`N?3p4Cj;o50A* z5XNb6sA=Ikwx0<-k!%V~V81}$+YDGuqg?)b8U2}8t+$A1vC)Bw<@LlIM;M(v? z$@t$9CuQS09tKUPwL%Fktj@8#pAPygZg|(|a>vKu0S|)~zf~$5$7Ch}R*8lN0VO5| R9*rae&ZaNS%>F^n5&%8k@k0Or literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-a2e76792-9576-4276-8bc0-8a76cbeb4f9c-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-a2e76792-9576-4276-8bc0-8a76cbeb4f9c-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..29c881b5317838753c0c525844369007b8c480ab GIT binary patch literal 977 zcmeYdau#G@;9?VE;m{3W&<8R_xtJLk7=-vZbcHxLm;_ESN^Fq}V9*81h(Kg`fimnI z3<6OST>=3N+CT|mhy)K%f(F604nV4Bv+1NQaxwv_F`S=9{g@i@0 zGB9W{a569)&|$%Ev^4`mn+}sBgOF=RQ+g+(a#&M%gK%)AX{JipBo9}HE5ML*5)g@W zabaL!bYM1MWpH5I!pP{r=m0d_jmf1zo`C`6ne5cm0;qS+F)(Os)&Hxo>MIk&q$9k7 z4y`Ac53jaNe-^sEp@8{L2UE(Eq9v7w_f$(x-Pm>gO1CAe-=iZP)eP%reVSarrylm5 zMIML+81!XsJe9wd##MB1?!r$^W(_UIRqpRaEfBo@jh3hQKzeq~^KG1!7>j77-nMIeJ&zU0L_j|(HXK^z# zm@-=wIQbZ{vX(G`y_qnVp_P*%M3947;2*OT50E+zjC4kYEDc>HcMcvQc19*<7FITP zDfW18*U0$H6e)rDVAsG}o2pUk#{x8YW^-%Li1y-WhE5)BOkN=ytq Q8np(TO<$Oq{ezq(019m6;Q#;t literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-dcc6ed0d-38b1-4447-b3e6-5b1788fc948a-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-dcc6ed0d-38b1-4447-b3e6-5b1788fc948a-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..fb4c9d71292472acd2da1c56b132398ba0c3d89e GIT binary patch literal 755 zcmeYdau#G@;9?VE;b074&;~MvxtJLk7=(B@7=_q4Lpi z5=&B3gkTH_z5oVx1_nO{5il7ARQ(?$;Kty9B)oy2fdQ%$ELX6Ofk9)d{$GU!rA!P$ zF5G+puA7-RD3wY3cuZF6Ofu;=(VuaB^2N1QrN?46hzoCLQa@nr{6Tzm^ugw*tF)!?3p|!>~k0%9h9BGpTf4OVb8T`s^_34cYS4I_{1V4wZJs@fB&@(yF$ZC z&!_+S{o_7$d=b9w#^GVqwT4%IDvJQC0ME|j7k@m`|LgQfru*?_;R#zeu2l7##kDCe zE>`Dx#-(2sKONcF8FWIV_gb~^wi!u1VeZM>WXYz-1olJ1Oom(zh9rM0BZu5OhtF7) zNgUo_*!Sq=gUhP=QjFQ&ECpWylQs-=EnU~)$gRNE)dbX{33ehWi3hfn;H>jk%fEAOr(_4 zglDqs`WbfaK!Mu1f(!#I1)Vmp>f zw{FYf^cHk^{&*7WkG40giU-SNmzO*AdEe*bc|Km=hv%)WFB1Vc zS|VA^2Nnd=I0679J*!D9OACvbd|vfI9%51xOfq5^7KAo=NcO=_L~KhCJR)dBfDBuF zAO%u>gZ<|WN&ZOqpa{*#O3X-OH6?{3gPrXWiEIglB61W7b^|oY7|9D zRKy%fDlsaeQQRhqI)JDPa00@QhA6t5Q$CRvUbVoi@K z9WNXo6OnXW-~zy*n463OR?IY^LbU=EV^LI+F8dgOJyFB893rYvPiNd)YjhM_rAgYO_7-|o;A$kV}H zfhB}5gn>tG9+T&>?66r=_9AZ?3@1p6W(-WA%CkQZ4jP0&gQu?1<1O<9yp1)bo_a%4 zpsA#~(G#eu@Rl3!NVfq;GODY&y)$GWx(^>!GFif%&gmkD!^hj(^20%ivsqnsm&2y1 z4n&fw&Eaxr_5w!%dSOmQLV7)BwW$a(cB|%&pMqQ1d;osMow6zTe#RZUk-vfWQ;CEp zBI_@5r*9cmzomG@yAS->5z3Zdn6K#)9+iiLZcaL35BIIY^`iHq8SEWgpya+jLBWAH P_gHoP-w2|%zD)iNqFd3* literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1dcd608-70b5-4d7f-afd2-676c97fb9fac-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-c1dcd608-70b5-4d7f-afd2-676c97fb9fac-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..df9e368a0c38933e6b4f08479d4facb64cdd1b9a GIT binary patch literal 946 zcmb7DT}V@57=F+H+3`5fIVm2ekvk3?gl;?A+;JrNx5~dc6=hx2DmQ9qI{ym0GR?4% zkg|)!o6sOK!Xk-4VyKj+bX62cKSURmNYT$lV9~cN0>O*AdEe*bc|Km=hv%)WFB1Vc zS|VA^2Nnd=I0679J*!D9OACvbd|vfI9%51xOfq5^7KAo=NcO=_L~KhCJR)dBfDBuF zAO%u>gZ<|WN&ZOqpa{*#O3X-OH6?{3gPrXWiEIglB61W7b^|oY7|9D zRKy%fDlsaeQQRhqI)JDPa00@QhA6t5Q$CRvUbVoi@K z9WNXo6OnXW-~zy*n463OR?IY^LbU=EV^LI+F8dgOJyFB893rYvPiNd)YjhM_rAgYO_7-|o;A$kV}H zfhB}5gn>tG9+T&>?66r=_9AZ?3@1p6W(-WA%CkQZ4jP0&gQu?1<1O<9yp1)bo_a%4 zpsA#~(G#eu@Rl3!NVfq;GODY&y)$GWx(^>!GFif%&gmkD!^hj(^20%ivsqnsm&2y1 z4n&fw&Eaxr_5w!%dSOmQLV7)BwW$a(cB|%&pMqQ1d;osMow6zTe#RZUk-vfWQ;CEp zBI_@5r*9cmzomG@yAS->5z3Zdn6K#)9+iiLZcaL35BIIY^`iHq8SEWgpya+jLBWAH P_gHoP-w2|%zD)iNqFd3* literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-d9aabc21-f287-48ae-baea-5b6d70640f0d-0.orc b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-d9aabc21-f287-48ae-baea-5b6d70640f0d-0.orc new file mode 100644 index 0000000000000000000000000000000000000000..df9e368a0c38933e6b4f08479d4facb64cdd1b9a GIT binary patch literal 946 zcmb7DT}V@57=F+H+3`5fIVm2ekvk3?gl;?A+;JrNx5~dc6=hx2DmQ9qI{ym0GR?4% zkg|)!o6sOK!Xk-4VyKj+bX62cKSURmNYT$lV9~cN0>O*AdEe*bc|Km=hv%)WFB1Vc zS|VA^2Nnd=I0679J*!D9OACvbd|vfI9%51xOfq5^7KAo=NcO=_L~KhCJR)dBfDBuF zAO%u>gZ<|WN&ZOqpa{*#O3X-OH6?{3gPrXWiEIglB61W7b^|oY7|9D zRKy%fDlsaeQQRhqI)JDPa00@QhA6t5Q$CRvUbVoi@K z9WNXo6OnXW-~zy*n463OR?IY^LbU=EV^LI+F8dgOJyFB893rYvPiNd)YjhM_rAgYO_7-|o;A$kV}H zfhB}5gn>tG9+T&>?66r=_9AZ?3@1p6W(-WA%CkQZ4jP0&gQu?1<1O<9yp1)bo_a%4 zpsA#~(G#eu@Rl3!NVfq;GODY&y)$GWx(^>!GFif%&gmkD!^hj(^20%ivsqnsm&2y1 z4n&fw&Eaxr_5w!%dSOmQLV7)BwW$a(cB|%&pMqQ1d;osMow6zTe#RZUk-vfWQ;CEp zBI_@5r*9cmzomG@yAS->5z3Zdn6K#)9+iiLZcaL35BIIY^`iHq8SEWgpya+jLBWAH P_gHoP-w2|%zD)iNqFd3* literal 0 HcmV?d00001 diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 new file mode 100644 index 000000000..ff018d173 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "pt", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 2, + "name" : "score", + "type" : "INT" + }, { + "id" : 3, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 3, + "partitionKeys" : [ "pt" ], + "primaryKeys" : [ "pt", "id" ], + "options" : { + "bucket" : "2", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "orc", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866469580 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST new file mode 100644 index 000000000..7813681f5 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 new file mode 100644 index 000000000..23aca360d --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "uuid" : "2337084c-4fae-4c87-869c-1794e79f560c", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-507ec8c8-79e9-40d0-a05d-4a402f7726a6-1", + "deltaManifestListSize" : 1118, + "commitUser" : "75a0f162-3926-40b4-b6ec-aa5da579c3c3", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866469655, + "totalRecordCount" : 200, + "deltaRecordCount" : 200, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 new file mode 100644 index 000000000..106e9b363 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "40f471de-0c7c-4a98-80d3-f36f726ae007", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0", + "baseManifestListSize" : 1118, + "deltaManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0", + "commitUser" : "f8c33e9f-0bff-4bdc-9b90-67e951ad7421", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469757, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 new file mode 100644 index 000000000..c1aa92c65 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "95913adb-0e78-48df-ab27-3566a670199c", + "id" : 3, + "schemaId" : 0, + "baseManifestList" : "manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-0", + "baseManifestListSize" : 1159, + "deltaManifestList" : "manifest-list-cf15e41d-848f-42d5-b0af-d75ffef9386f-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0", + "commitUser" : "c31fe56e-a1d2-4a59-aeb8-ce3be0915349", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866469817, + "totalRecordCount" : 202, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 new file mode 100644 index 000000000..136e05854 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "a0fa8d3a-44e1-4b98-bc68-c8b89bf3da42", + "id" : 4, + "schemaId" : 0, + "baseManifestList" : "manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-0", + "baseManifestListSize" : 1196, + "deltaManifestList" : "manifest-list-953de685-b6f6-427e-bd84-42fe41e9ac1f-1", + "deltaManifestListSize" : 1122, + "indexManifest" : "index-manifest-cb21ca4f-0678-48b5-b2ba-f797e7dedab4-0", + "commitUser" : "07771156-18a8-4384-b947-4412c4bf53e6", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469903, + "totalRecordCount" : 203, + "deltaRecordCount" : 1, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 new file mode 100644 index 000000000..3ad236ffc --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "e255e37c-142e-4888-812b-05a76e52f996", + "id" : 5, + "schemaId" : 0, + "baseManifestList" : "manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-0", + "baseManifestListSize" : 1234, + "deltaManifestList" : "manifest-list-b28d8d46-edd0-4e5c-b499-20feaf7d0fa0-1", + "deltaManifestListSize" : 1126, + "indexManifest" : "index-manifest-55558948-e419-4c96-912a-f7d4b1d25af6-0", + "commitUser" : "3a3579f6-c63b-4621-b1dc-c08a781f0181", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469992, + "totalRecordCount" : 201, + "deltaRecordCount" : -2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base new file mode 100644 index 000000000..106e9b363 --- /dev/null +++ b/test/test_data/orc/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "40f471de-0c7c-4a98-80d3-f36f726ae007", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-0", + "baseManifestListSize" : 1118, + "deltaManifestList" : "manifest-list-ba668ee4-c6c5-4340-ab1a-2be9b3a2c8ee-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-2811d607-fabd-43f1-bc76-82a2051280a0-0", + "commitUser" : "f8c33e9f-0bff-4bdc-9b90-67e951ad7421", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866469757, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/README b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/README new file mode 100644 index 000000000..43ba26ffb --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/README @@ -0,0 +1,24 @@ +id:int score:int tag:string +primary key: id +no partition key +bucket count: 1 + +Generated by Apache Paimon Java release-2.0.0. +file format: parquet +deletion-vectors.enabled: true +deletion-vectors.merge-on-read: false +write-only: true (explicit fixture commits produce snapshots 2, 4, 5) +primary-key btree index: score +target-file-size: 8 kb +compaction.force-rewrite-all-files: true + +Snapshot semantics: +snapshot-1 APPEND: ids 1..2000; score=id%10; even tag=keep, odd tag=drop. +snapshot-2 COMPACT: full compaction builds source-backed score BTree payloads; source-file counts per payload are [2]; 2000 rows. +snapshot-3 APPEND: insert (2001,0,keep) and (2002,5,late_drop); indexed compacted files and a visible unindexed APPEND-source file coexist; 2002 rows. +snapshot-4 COMPACT: delete id=10 and update id=20 to (77,updated), then lookup compact in one fixture commit; deletion vectors coexist with indexed files; 2001 rows. +snapshot-5 COMPACT: full compaction rebuilds the BTree source groups; 2001 rows. + +Fixture construction note: with deletion vectors enabled and merge-on-read disabled, Java and C++ batch scans skip literal level-0 files. Snapshot 3 therefore promotes Java-prepared files to the maximum manifest level while retaining FileSource.APPEND. The source-backed PK BTree policy indexes only COMPACT files above level 0, making these files visible but deliberately unindexed. Snapshot 4 publishes the Java-prepared compact outputs and index changes while omitting their transient, never-active append inputs. These are test-fixture constructions, not ordinary end-user write workflows. + +At snapshots 4 and 5, predicate score=0 AND id<=1500 returns ids 30,40,...,1500; id=10 is deleted and id=20 has score 77. diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-0caae41e-5153-4d29-8a48-dbbdf015f77a-0.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-0caae41e-5153-4d29-8a48-dbbdf015f77a-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..17dcdd3d01be29f491d90ff05000cc762e6014c2 GIT binary patch literal 1322 zcmb7E&1(};5T9;dk}XDTvGdrKMfQ>98c zK}tc;e?UA|^jOb=Cr?ESejpz8AP9vXdMLGZ-hL#h^`J{O@4fl>&CG9R-^B2wiGW?0 zz@4q1-y#|^I?5;mBwn5W1tS*$-bVp8>Fod@osJn?|FAQB#PR{aJF*gw@DPviV1%aQ z*4D;zmPH9!0Ia(fJ}5&Czb`*p?i8rPqZz-8%tBt||43590sQgh#S<3~Fw(-S14sfg zB*KU3^{`Wz$Gwf$pF&<)Z4RFU2g)4oI!?Wl)@pIVvFt{zPTp#_q$*65OpD+NC6)VP zIbcLpIY2 zErcasy2@~gwkkg*f-H?3uNNz`wK;3Pe8-tBVvMA5P@>jY?xt0-3;PEL2T|%W$-1l? zx$A{o{&LRB7p`5(O&0b@dBSrzW=#!`7jo9v)%=zC>A_v0(O#aEyWL{QTOu$~S6ouU z^W|E@DM)}ZI)xUBx8pF#G>uF-Q#G|}fJV8+W#TSK1Y?}>AwFfuS+5ITOE@Ilo_Hss zf+m=PK$f7^qu9eAhOrj}9`o>y6yXAobK98CwtA>E8%qJ2ooz`E2YM?4t%q^Q4Md4E zSykEzabw)#k2{PN$#~PDc@yFNiwdZ4@J^tfb-xheMtoXcAz!1&`F2oJJMOU7D=@WdFlQf!?L*S;bX-bR2I^YKJaT{N378LKRt zf<$Y66(%OS4TeS$_){4*z8Ap^@Q)$R44Ln<9=c3M>&3>sMQ7pMY(u~AH0CR{Io<3v zQ@u$&QCh52ZG9lwSMJZ4$#Q?X&+faBO7=Tu#!RQpV%aIpIOTHMEIBhN^Nd=h5&Po@ LVY(?M=$8Bq{gK@p literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-14f3c2d2-86bc-4305-8ddc-3d7587b8e8ac-0.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-14f3c2d2-86bc-4305-8ddc-3d7587b8e8ac-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0564c8255b7f7d7b25df08e84a157eedea0a725a GIT binary patch literal 1269 zcma)+&ubG=5XUDk+hoPGw%B=OVUZvtM9`XTO;SrJwi>9`hFD3!UY2CDwSlCGN$bgj z=tVpTp1qY`l%9Gy#1=G!-S_6r`^a0HKR^8o<{|*20N?{4`!L4w)7{tK_q&|~cpoG``q{36*y-#%mJTBJ zyZuMtQ9Ox{o_%~XWYYrZ>B|E!!ufp*Ug+aEg>RpJcsQ`n`JU@6q^>tBr8Toca+MLL zge8%t^FpR@$1J;PDPeEfb1cy%aZ9{<%?HCIA(FhstH(R&NKwM>$~jcIj5$&Qipq;v zbXHqF&$!!NZZlkR95(#8BJ}ei5#`q0V6#-MH&*qwv0~Os7$R#F!a;pLdrhxa3cvUE z_JV*C*@pG2*(-%??tE6y6&7Z*i-n(bc*MSNSTD|8E@buji@6KYW651{qHZ0PN33k< zHY1owEXu+IZKKgL3k)zsJO5hgdK8k9s*)>bimKFNK;2QIGL^#>f+5a$isxc-#_opw z8-gGK6{=z)^l*!Z8p$$>!}#!U9k8Vyu|yHnF>+0A6dm*8V^{BG3y7 zr>#Ja@ZN%W;I58(42aOm~PqTMwB= z68L5iRen2wQ^3D4&SEmZBeINNYL;3z*3Gq3^;YDj*=koCs}XHXON}KW@$!1LR*9TR zj2jb^TEdt}k5|UerVsLQ7K9#mYT$IAfg{3ZBYjX zMueCcTL#1eNKCCL0}DbODkfCu%EZ7%2Zjm(<(@y{gfKwl$nV{|``!QDojf%nBGv6lp7?3=n^L<|mAe02~E?0s033=q!Rm`0@GXgPtzey8ycq`O)u#`*uP& z{psf8_q|;qBLG1=z@z_%5Dn9~w{`!kATelr^Adliw+RcB5dEQ)e zyEm7lDw3XUSY_S|tH6OuOK6`Fa#GqNb566ID%} zi-AVkqB2#^6M|tTvSBt8lk-jwoQ5!n+Z}OsR0^6wx#&HEixT5Y8St-ku!&P4YlN^~INu${;i-d>QF`0W6x=d+;Lt>)_eT_^OBgp6(V zxawUf6|u!y6aKjp8=UpoXJzCNmDXv4%^PQx*gog+$@8`OyH0I8ocA!~Y746lk2lh$ zU1O`U-C#Zoac^W5k&kd2gr*PJCkZw7CWL8V8$l+7QTuv{WR;6Z{W=^kIwXA+h RoujGx8n-$4dYJ literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-0.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8838eb1eba840f6be9f3a7e26ad9468feebbc99d GIT binary patch literal 6666 zcmeHMXF30r{Pa$q=Cs1%AV(VIy*jG7-@mLreNoB zH`L+grL0>l4XfE8;ZhG&BeVM+%PLPAmdjfyRjjbDTUJqI$l^USWEz(ctHoc=7tm&?h?Odb>d81Gj zP}5y1+c8Byk=r;(<(J*PRkl4?KVE*rLFFUtE|6`r)sNxcc%>Yq>5|Dx9_euTb&b-q zrfa!O-mDwWz5YPyk=?aeR(w(yDQ}>Zd|_9HY-^D&jN9;q64P|1$ch%}hRN$|lxR(7 zqHN0)-4Jg5KBY@`C&)Gj>-^<)B}yk@=XlvBTb(zzZjI7G(}Bx2KGF*0*RqxNH64+% z!e*@-_gbpbHoIe(Y{N;dtNiLTrM0lbUsh10b>v=sM`@{P7s%Ex(%Q+d2$kkF?XI$Q zQ?ypxD}$6~+3j|+wZU4Byw*X<61H<>`LVtL!Gn(&r!!v<#bC|Z^E4Tq$+@%icqNCuC9LFQ;8JE)t;1`xL(!OqU)9{#fPgs zsA9T7+*Pf+D_MX?s`=DbqQS4LSXU=W!^6}VRYcc!b!O?#OOo+nYLwbS)K_)J>kdg~ z;6v0d)MmO)+!>-Pl}y6@)lSqVqRy|=URNL)i+igbsEzcsu8w}~a!D*MP}@_5#I>r9 z7Hx(^guAJ2sSWg1aYwZ_Q4)l^s;#L4;;LUqv39(~2X|CkQtRm}UF})gNC_XeQ=3!k zh$~g?@mhb06K3;`R`&tHc`Ts997#QR~-kujNQsIHKOy*Y`{+`*75%)oZAJ z>8PW92D@J9fo|x6PUwJkcmQg+4{h)*d;?#@e}IOs;2zwCFX0Z{hFd^EE408(XoeC}0neup4$kDeQzDupPER3CN)swn7nXfz7Z9 zHbNn6fC5+#>tHS9!y5P(f(z#=tu;8b$#QBVh!@!f=RzXo!MHh=6bqK^O=j6hdGa1jA4Wf*~*%0wDnW z!4G`F2fSerc!4JfzysWY4{m?~4^VIg7jOnAa0Cb7f<4%QE!coHSb-&2fH`o$4A{T| z1l9z;Gjrc&D$ymr{POC(qiXiPN0s+A8ak!gi!1kztf@4^hSM=*3=xCH(9vWx5sgLD zQDhVmg+JG(|nRo@G(B^M!FGh zm>Z3e7=d9J%_DgP5984&i4rJ=(ypW{;flG^E~E?Lg1OMnq%+}+Inz$06XAq8(T=1e z;fOiX4x|I&fH}}yl1p$gE^SZR6ZV)rZAaP>c9$?m?dpNS`ZeP1#M266XuvX%^^7i2jkFYq#0p`nbB;LO|UUG%_3O@ z3uDm;i4X{e(EFOHdwreD?#0nh6pt0(D;_F(72S$XMY}?+Xj6Qn_>bZ%#a+c6#Vtjv zg3UsnUrV*P$3hS$_2%cHyLsqO_1zE5{+f(m1|vhKB{~N>=3n-B`RY%!#$wO&KQg{p zY14$fLT0iA>NABATRUt&P+oQ2d#1q8uETPq@`~MuXV$rFZ8!5!)^fbgOgpug^!U7@Q-rwMe|?1dHDE9MRY?H9j1Dc4+j*6N}8y#I9h6IGAg<&V$Sb3Go{TzXRWsQN;y`=gW9r^~)Sepv4Q{n5%LW#1j%HP!vQs^ZYH zhX*!r-5>6A`1Hd0PtKh^Q**ld)X5Xaj~zX7_)yis1N-;wt*j_7Q|uvk?<(E7WBaxe zdGXewEt@xOEZk7Ae%;#qHUG+!tzIRSto(Sz@@2V8bF!CYEnc*6!TgWrWoC%e)8?jr zI49+U*~znr_mdK5Cd`;VZE8GBnLKIYdlTZuzdLU1n0H>aeKcB=Oi3FImHjCnji8cS zmlhhz@3+m1u1RiLFu|Z`NtqX3l6*7G!a!bX%M8s|FG(opH zJ|*3`WLBMzg>K`hwvO=hT1m%2^CEO;!v5iD2RWOae8=Vg)PBHhqrH3hT;<#Y>$?%!*vzjUGcaOJMz4S6fFGUp`4$Bl{#3GhVi z&3}3NxV!D{&AN-{PaUc#-C7`9zBnT#5ys<@!~6xVcAS@nM_u23*?jHuPfk{q?<`us zdf6iJ2Q#O}UP=o!{JPy879<69>z7Y+1J|cVYVMgvsMZgb(#~ceY`_ zV14GWZ`3M%yGneNTN#_%Su5>2VqYG$@Xr2N@ewByVDb*{*z`kI+kM8ZL!Nh3S(bQ?$Zc1qRaq4C|L^{=e>Etv@TQw6tyZSH zv!R(6l6)rRY{G^n*Pt2u@uhbP8l3|sZ4-zR3T`<1jLm1H4*VWsgLDMV3W~{5Ta`G7?)81Np&BC4^mU3qL+N&IECvQCGR{j;XxwWGpYf_uc3+JVOIQ#vC zsgoy+8$BXAJalNFua`T|nQLRg{`L7YUH^AI9rwS!ce|yr{>mlQg|pSi458RU3o_H@Bqz;?pA66_^_!By+xmZ{ZoZ)k#Xlxd z#6KqSyuYJfl`qHmlIAe=kvFQ*v>xZ+C>!PcRw{X$$|KWz+yeDNpYA)l+4A*U+&Ja; zlFE%4cgw*v>}~D!w)T2kd%dl_-qv1kYp=Jp*Z*tn#X+Z|^U?czo4T!W)b&kTAHzbJ zT#c}qX2vuo-6Q5qGky%UWEzv!5o@Ly$&}MyTA~T)fo3G)@c?JUv4aEq3vYBdTCuxsCl_Zn8^7-_@=d}saSV)RXoib_a z_yDh{7se-oQ!Z9cudQKbvl%d>VF7G6^JU`f+{`34g4!`Xv*l?Y1t3uzfq)sxyXC+a z@D~Rn0!Fz2E5>`jX@c4@&v;Kq^UAyet#DH)OmX2MLtY2Q)MI?Yd>>#0AvgoKe>TeF zy(G|3#0beTLH@-M6ZZK$UK20ypcnoom~OvEm=W@Oh~IGkEbPv z#5nekzR8dN$Ij_C>bpDncep5yl?8twIwr_N*1R3;z+*{)Wr>Wg#+CU-*EhYt)%B$j zyzVz}JH83r7;F=Cp&9QMD{u=DcI;8Nq`d CTxR_M literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-1.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-6d915c8e-2c9f-46a7-ad5d-4423f7cc5d59-1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4403ca09ddd41c2ef8cac88ae8ce69987c5ad694 GIT binary patch literal 6570 zcmeHMd7RYNeb>xr?t9)yZ)P<+Z)WGdk3DcwitWUk5FgY*USz7K3WhZ%F@}KcUZLwc z&f%nSX_Q8p3Rba84I#vF9ZW2D<5HKpgb-L;fDponU}83eVmEQ!tdF=FW^HJld_MU* zMmuk%_j~Vsf9K5nzMr}A>cuSrBX%z!&LZMFk9Y2MCWz^vDRB`l3@bj~coSQA{gs&K zwT&nenrVOAf?c>O*Oq>g&)Q$NylnpQBlHLuQHJ?pdKe5VL;Mgu1csDBevlpngG!rk z({0dJT6~LcftJ$bn{*R2l?LCS8=#@o`8r((b*0AF=o+XgRlZ7BK~<^n6}kc{N|`Uy zWl&a1e2FfBl2YW0bP*Jl0$-pDprGXWJe>!5CCBIJ9LOnIK1*joR>|-gIs-Dw06#zv zfB_}Vr|C3ED=9ujr$90HyTteRLn_ zQ{sG_j)S-o<70FT#1zOw8Um=0JV}#)R0y7+2|y@OK1xSHQ~^Ao0RT#bkI)eiQNnzf z4uh}~;zM)@gp?p3q=O)+1o!|Q00G6%`)NP$D?Z*w`+!gJ@?P2tyo!hS&>rAX+`OB1 z1GnPhU9=0h6esVboxrI$cn9qO4#m#fX*;khIFHjfz!e*Bqiw*ZSa~aL1y;qvTWAZg zC>W2?7{HVpMh;)296Yr0)Zm+ECMV9#Zk?Q%nO!+KaeB5eIdO9K%;t&Xvs*V$9GzXc zdE(G)Ve`cF?3qmy`)9Xqn%FzLa?`|~*}^6`tPTl7%n%$>2Zcdq5Duzsq0O{mTWtv~ zrUhGSQ)n_x*i;)rgK5BqS{Ld}9oE&FP-AMardEY2Q-xKvB2<_Ptf*z7%#>kSEeR#2 z1WRgBC^AJ@R0~3ZDZql77xGLV=GB~#V{$O3W`!)1g;_NtWS9)hr~|?PGXMwFw2)@f zFs-J96qAA}6$ywz5UEKa$s}P??HBr)e%P-jgang-36&BkhJuvZC-gCWuuqK(aV8Gq zYD|bRF&I;!02v6ON(v-HLQ*9Jf*~NGMujL7g;5mqR z=u(}6lW{_)>JS`^13FZ@U}x;muHpjD;1E}Bf{n33n`#xTj1^i{i(p|a(4t}j#$XUr zKX&)wQT5>AGpE*XKHoXlIop}(yx2M2In_DYIng=ZIo3JaInp`Qd9E|vIndeP+1J_I zd8V_c^K@r-=c&$A=gH0!on3BmkR7B3^|sh%+f-X`i7mE8we+UgWSdk|Z-@=HK{fQc zSZC{0U9X8Xwno+Ts#s;KR8_Bt6}CcE^s-oH%T!q}i6yo~mGq)mWQ$Z$FNg)UKo#`7 zm}m1;UeAd+Hb>?3te9o9R94T388$;@^Z{{z9iRsEw3ueoR9a7oDKCeUwl4ieA=Bd3BHIVLg;bcZ+V;O}TZK=we-zOLvM+)=4>ahv;A( zltZ_RcGgbWbzH<*oWgaRXk%@ZO}C0x)=F7*i)djjltsrxjKwHSU%l(_hxCI-R=)7v z@1K{?$!Fym`9=A(d`dnkpOBBs$K<2(5&4k(oIEWbkoU{`7E1{B6l!{yt6^(*a;0mZ<JE0xd zj%i1=BibSDIc-`ypzYW8X?wM2v_0C>+HUPBZAyDmdqUf#?bIIA9@QSvc4*tRZQ8@y z7VRPJ8``8BGr!->FaLWs5C=lUk57Dg2mbDM;CKqV>B<=!_S3UJJNKXGUwZk!e*TMB z{`=Mc`Q_|uufOr;uinCNi`9nP9Zr|qJNPIn$Zt^c;QE`z3%!O7X87EA6-n}G`57f zdFdarfAq0G=03jcmgOtPZ(VuYs!x3K_SJW+S^KGV>+ihl?mzj{Kl}5)`1FRqoVe$& z?)}VtpZ(nDzi|H-zx3B%e&BC5{_R)zum0WFgs*r0Ufi^KQu@Y&5B9Ozbe0~qSY z!$;Jk+A;n3^CygxKR)%s>7TrKX682q!2VtjfaAR$0O#*m0Gz$9|1)Q{MNS^HoZ@Ve zg{fcK7Ps%eFM0!+9_2?DKD#3Py=NUSf5P7V0KrZ6kRo1s>A*Mf`Gl3g-zE~N9!&2u z#z7>BSEQ}?_XtP#$UB51X6~B~_O@MSCYPDXWoB}jnOtTjmzl|BX7V0qCOB~;aU1dc zwCM_Ajgxa?mr21cT=AIe;9P@t$#PeH_f+?Gsrc7}y}i2|rAv2r9Wg<= ze$0szq>GqLnMPpo1o?1^xHmlae7;32ntPmTU)(Z76Zcu>?H7+`XWy<kdV4JcKM~{AVY;@5FN4Z5~ zH+*1p@z@*YVz76>%PqP3`ms^&x{oZnCYKz3t-F%g0bJn=-k7JYCRo?8oHb@&zkJQQ z@i8k#_{^E@&Sfie*f5?*m|Mx|U_24OJ&z?!<8m%DdT(1o_{=r(l?CKvD({-#OGPg) zBv!nTnBL5ciOHT#Mc8~3?&Qq`Nlq;x>R1;N??JxHkskKW4dlJy{KKjF7n2!#C&Q+a zcT#*&`hS(ZdNUCt&n_VrTu^4(kGtZrO9Ru>vx5ApC4at^-_}a>YNPkR>xLNttT)5r z)oxLnZkWDwZ(VO5SZaE+jhXXiEM=rt*<1R;hE;~KQEpGBF z*8B{eKepJkwOg4NZM``AwYE-m!SB0y&Q@Q%2i{F~5BhU9a@vyLHZGNNqE6x7`YCxF30r{Pa$q=Cs1%AV(VIy*jG7-@mLreNoB zH`L+grL0>l4XfE8;ZhG&BeVM+%PLPAmdjfyRjjbDTUJqI$l^USWEz(ctHoc=7tm&?h?Odb>d81Gj zP}5y1+c8Byk=r;(<(J*PRkl4?KVE*rLFFUtE|6`r)sNxcc%>Yq>5|Dx9_euTb&b-q zrfa!O-mDwWz5YPyk=?aeR(w(yDQ}>Zd|_9HY-^D&jN9;q64P|1$ch%}hRN$|lxR(7 zqHN0)-4Jg5KBY@`C&)Gj>-^<)B}yk@=XlvBTb(zzZjI7G(}Bx2KGF*0*RqxNH64+% z!e*@-_gbpbHoIe(Y{N;dtNiLTrM0lbUsh10b>v=sM`@{P7s%Ex(%Q+d2$kkF?XI$Q zQ?ypxD}$6~+3j|+wZU4Byw*X<61H<>`LVtL!Gn(&r!!v<#bC|Z^E4Tq$+@%icqNCuC9LFQ;8JE)t;1`xL(!OqU)9{#fPgs zsA9T7+*Pf+D_MX?s`=DbqQS4LSXU=W!^6}VRYcc!b!O?#OOo+nYLwbS)K_)J>kdg~ z;6v0d)MmO)+!>-Pl}y6@)lSqVqRy|=URNL)i+igbsEzcsu8w}~a!D*MP}@_5#I>r9 z7Hx(^guAJ2sSWg1aYwZ_Q4)l^s;#L4;;LUqv39(~2X|CkQtRm}UF})gNC_XeQ=3!k zh$~g?@mhb06K3;`R`&tHc`Ts997#QR~-kujNQsIHKOy*Y`{+`*75%)oZAJ z>8PW92D@J9fo|x6PUwJkcmQg+4{h)*d;?#@e}IOs;2zwCFX0Z{hFd^EE408(XoeC}0neup4$kDeQzDupPER3CN)swn7nXfz7Z9 zHbNn6fC5+#>tHS9!y5P(f(z#=tu;8b$#QBVh!@!f=RzXo!MHh=6bqK^O=j6hdGa1jA4Wf*~*%0wDnW z!4G`F2fSerc!4JfzysWY4{m?~4^VIg7jOnAa0Cb7f<4%QE!coHSb-&2fH`o$4A{T| z1l9z;Gjrc&D$ymr{POC(qiXiPN0s+A8ak!gi!1kztf@4^hSM=*3=xCH(9vWx5sgLD zQDhVmg+JG(|nRo@G(B^M!FGh zm>Z3e7=d9J%_DgP5984&i4rJ=(ypW{;flG^E~E?Lg1OMnq%+}+Inz$06XAq8(T=1e z;fOiX4x|I&fH}}yl1p$gE^SZR6ZV)rZAaP>c9$?m?dpNS`ZeP1#M266XuvX%^^7i2jkFYq#0p`nbB;LO|UUG%_3O@ z3uDm;i4X{e(EFOHdwreD?#0nh6pt0(D;_F(72S$XMY}?+Xj6Qn_>bZ%#a+c6#Vtjv zg3UsnUrV*P$3hS$_2%cHyLsqO_1zE5{+f(m1|vhKB{~N>=3n-B`RY%!#$wO&KQg{p zY14$fLT0iA>NABATRUt&P+oQ2d#1q8uETPq@`~MuXV$rFZ8!5!)^fbgOgpug^!U7@Q-rwMe|?1dHDE9MRY?H9j1Dc4+j*6N}8y#I9h6IGAg<&V$Sb3Go{TzXRWsQN;y`=gW9r^~)Sepv4Q{n5%LW#1j%HP!vQs^ZYH zhX*!r-5>6A`1Hd0PtKh^Q**ld)X5Xaj~zX7_)yis1N-;wt*j_7Q|uvk?<(E7WBaxe zdGXewEt@xOEZk7Ae%;#qHUG+!tzIRSto(Sz@@2V8bF!CYEnc*6!TgWrWoC%e)8?jr zI49+U*~znr_mdK5Cd`;VZE8GBnLKIYdlTZuzdLU1n0H>aeKcB=Oi3FImHjCnji8cS zmlhhz@3+m1u1RiLFu|Z`NtqX3l6*7G!a!bX%M8s|FG(opH zJ|*3`WLBMzg>K`hwvO=hT1m%2^CEO;!v5iD2RWOae8=Vg)PBHhqrH3hT;<#Y>$?%!*vzjUGcaOJMz4S6fFGUp`4$Bl{#3GhVi z&3}3NxV!D{&AN-{PaUc#-C7`9zBnT#5ys<@!~6xVcAS@nM_u23*?jHuPfk{q?<`us zdf6iJ2Q#O}UP=o!{JPy879<69>z7Y+1J|cVYVMgvsMZgb(#~ceY`_ zV14GWZ`3M%yGneNTN#_%Su5>2VqYG$@Xr2N@ewByVDb*{*z`kI+kM8ZL!Nh3S(bQ?$Zc1qRaq4C|L^{=e>Etv@TQw6tyZSH zv!R(6l6)rRY{G^n*Pt2u@uhbP8l3|sZ4-zR3T`<1jLm1H4*VWsgLDMV3W~{5Ta`G7?)81Np&BC4^mU3qL+N&IECvQCGR{j;XxwWGpYf_uc3+JVOIQ#vC zsgoy+8$BXAJalNFua`T|nQLRg{`L7YUH^AI9rwS!ce|yr{>mlQg|pSi458RU3o_H@Bqz;?pA66_^_!By+xmZ{ZoZ)k#Xlxd z#6KqSyuYJfl`qHmlIAe=kvFQ*v>xZ+C>!PcRw{X$$|KWz+yeDNpYA)l+4A*U+&Ja; zlFE%4cgw*v>}~D!w)T2kd%dl_-qv1kYp=Jp*Z*tn#X+Z|^U?czo4T!W)b&kTAHzbJ zT#c}qX2vuo-6Q5qGky%UWEzv!5o@Ly$&}MyTA~T)fo3G)@c?JUv4aEq3vYBdTCuxsCl_Zn8^7-_@=d}saSV)RXoib_a z_yDh{7se-oQ!Z9cudQKbvl%d>VF7G6^JU`f+{`34g4!`Xv*l?Y1t3uzfq)sxyXC+a z@D~Rn0!Fz2E5>`jX@c4@&v;Kq^UAyet#DH)OmX2MLtY2Q)MI?Yd>>#0AvgoKe>TeF zy(G|3#0beTLH@-M6ZZK$UK20ypcnoom~OvEm=W@Oh~IGkEbPv z#5nekzR8dN$Ij_C>bpDncep5yl?8twIwr_N*1R3;z+*{)Wr>Wg#+CU-*EhYt)%B$j zyzVz}JH83r7;F=Cp&9QMD{u=DcI;8Nq`d CTxR_M literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-1.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-dba92a1d-4256-4c89-a3e9-51bedc74f336-1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4403ca09ddd41c2ef8cac88ae8ce69987c5ad694 GIT binary patch literal 6570 zcmeHMd7RYNeb>xr?t9)yZ)P<+Z)WGdk3DcwitWUk5FgY*USz7K3WhZ%F@}KcUZLwc z&f%nSX_Q8p3Rba84I#vF9ZW2D<5HKpgb-L;fDponU}83eVmEQ!tdF=FW^HJld_MU* zMmuk%_j~Vsf9K5nzMr}A>cuSrBX%z!&LZMFk9Y2MCWz^vDRB`l3@bj~coSQA{gs&K zwT&nenrVOAf?c>O*Oq>g&)Q$NylnpQBlHLuQHJ?pdKe5VL;Mgu1csDBevlpngG!rk z({0dJT6~LcftJ$bn{*R2l?LCS8=#@o`8r((b*0AF=o+XgRlZ7BK~<^n6}kc{N|`Uy zWl&a1e2FfBl2YW0bP*Jl0$-pDprGXWJe>!5CCBIJ9LOnIK1*joR>|-gIs-Dw06#zv zfB_}Vr|C3ED=9ujr$90HyTteRLn_ zQ{sG_j)S-o<70FT#1zOw8Um=0JV}#)R0y7+2|y@OK1xSHQ~^Ao0RT#bkI)eiQNnzf z4uh}~;zM)@gp?p3q=O)+1o!|Q00G6%`)NP$D?Z*w`+!gJ@?P2tyo!hS&>rAX+`OB1 z1GnPhU9=0h6esVboxrI$cn9qO4#m#fX*;khIFHjfz!e*Bqiw*ZSa~aL1y;qvTWAZg zC>W2?7{HVpMh;)296Yr0)Zm+ECMV9#Zk?Q%nO!+KaeB5eIdO9K%;t&Xvs*V$9GzXc zdE(G)Ve`cF?3qmy`)9Xqn%FzLa?`|~*}^6`tPTl7%n%$>2Zcdq5Duzsq0O{mTWtv~ zrUhGSQ)n_x*i;)rgK5BqS{Ld}9oE&FP-AMardEY2Q-xKvB2<_Ptf*z7%#>kSEeR#2 z1WRgBC^AJ@R0~3ZDZql77xGLV=GB~#V{$O3W`!)1g;_NtWS9)hr~|?PGXMwFw2)@f zFs-J96qAA}6$ywz5UEKa$s}P??HBr)e%P-jgang-36&BkhJuvZC-gCWuuqK(aV8Gq zYD|bRF&I;!02v6ON(v-HLQ*9Jf*~NGMujL7g;5mqR z=u(}6lW{_)>JS`^13FZ@U}x;muHpjD;1E}Bf{n33n`#xTj1^i{i(p|a(4t}j#$XUr zKX&)wQT5>AGpE*XKHoXlIop}(yx2M2In_DYIng=ZIo3JaInp`Qd9E|vIndeP+1J_I zd8V_c^K@r-=c&$A=gH0!on3BmkR7B3^|sh%+f-X`i7mE8we+UgWSdk|Z-@=HK{fQc zSZC{0U9X8Xwno+Ts#s;KR8_Bt6}CcE^s-oH%T!q}i6yo~mGq)mWQ$Z$FNg)UKo#`7 zm}m1;UeAd+Hb>?3te9o9R94T388$;@^Z{{z9iRsEw3ueoR9a7oDKCeUwl4ieA=Bd3BHIVLg;bcZ+V;O}TZK=we-zOLvM+)=4>ahv;A( zltZ_RcGgbWbzH<*oWgaRXk%@ZO}C0x)=F7*i)djjltsrxjKwHSU%l(_hxCI-R=)7v z@1K{?$!Fym`9=A(d`dnkpOBBs$K<2(5&4k(oIEWbkoU{`7E1{B6l!{yt6^(*a;0mZ<JE0xd zj%i1=BibSDIc-`ypzYW8X?wM2v_0C>+HUPBZAyDmdqUf#?bIIA9@QSvc4*tRZQ8@y z7VRPJ8``8BGr!->FaLWs5C=lUk57Dg2mbDM;CKqV>B<=!_S3UJJNKXGUwZk!e*TMB z{`=Mc`Q_|uufOr;uinCNi`9nP9Zr|qJNPIn$Zt^c;QE`z3%!O7X87EA6-n}G`57f zdFdarfAq0G=03jcmgOtPZ(VuYs!x3K_SJW+S^KGV>+ihl?mzj{Kl}5)`1FRqoVe$& z?)}VtpZ(nDzi|H-zx3B%e&BC5{_R)zum0WFgs*r0Ufi^KQu@Y&5B9Ozbe0~qSY z!$;Jk+A;n3^CygxKR)%s>7TrKX682q!2VtjfaAR$0O#*m0Gz$9|1)Q{MNS^HoZ@Ve zg{fcK7Ps%eFM0!+9_2?DKD#3Py=NUSf5P7V0KrZ6kRo1s>A*Mf`Gl3g-zE~N9!&2u z#z7>BSEQ}?_XtP#$UB51X6~B~_O@MSCYPDXWoB}jnOtTjmzl|BX7V0qCOB~;aU1dc zwCM_Ajgxa?mr21cT=AIe;9P@t$#PeH_f+?Gsrc7}y}i2|rAv2r9Wg<= ze$0szq>GqLnMPpo1o?1^xHmlae7;32ntPmTU)(Z76Zcu>?H7+`XWy<kdV4JcKM~{AVY;@5FN4Z5~ zH+*1p@z@*YVz76>%PqP3`ms^&x{oZnCYKz3t-F%g0bJn=-k7JYCRo?8oHb@&zkJQQ z@i8k#_{^E@&Sfie*f5?*m|Mx|U_24OJ&z?!<8m%DdT(1o_{=r(l?CKvD({-#OGPg) zBv!nTnBL5ciOHT#Mc8~3?&Qq`Nlq;x>R1;N??JxHkskKW4dlJy{KKjF7n2!#C&Q+a zcT#*&`hS(ZdNUCt&n_VrTu^4(kGtZrO9Ru>vx5ApC4at^-_}a>YNPkR>xLNttT)5r z)oxLnZkWDwZ(VO5SZaE+jhXXiEM=rt*<1R;hE;~KQEpGBF z*8B{eKepJkwOg4NZM``AwYE-m!SB0y&Q@Q%2i{F~5BhU9a@vyLHZGNNqE6x7`YCx528a@djgf#@1OoCx#kqEMeEh=C|#j5>W5OJ(s5CNkil7LELweI_hpi^~> zt+mz`t+lPvWwb7c8-j{DYB%=umi7*dSBYJ`6NAMrKkmQ#qt27eob^5Lw`9(FpG-)Z zsKF5S=@9I;AGYhad$m#(RwLzGB$x!DkYiE5qEVtmbjdo$&&%`s*>Pg@_zLqHcH!F? z$1=fH0Vk*}G7+^6*2H4VPxPK6Mf1%y*Q|rImKK`cQe+@%PFuAPTEC%p7Zl~2uN<|; zSG3-wca1NaMO=By8e80Yh29xelxwaov1+uf=jb=2MOj4kGHcX>mJ>9^6~1D=Y_LXD zv>c;%)E2%-T+XtF6}KFww;w6YFkecuhH6{hq06=urV*D?toRPu_+muUh++xrWRj0f02aT`L8wv~&=5t58v5LkF zdi{7qC~@wsZtvp8vGlqqL$JBBq+6zKOr_UK4FN>uvTo^vhGcpTr}s0THFUdIG{n=Z zYxPRvY*x2xaYF>X>WCgQpGoU>);0vwE4S!n#F>r`WWd8A|wbTP)|>nmRh}8y7N31i@|AoiA^?*3oJ# zWCju{Zj+@dujy#VE#qt^j_~C+>Z-z;N;;~H)0t>O!ELadYil%g95+s7!U;LIUU#m% zF|Ff(aUv5!_;Bkim3fVtjvdA^Oh3YlTdS)KYm{}YHV$L_2~Tc~&oIU?3c`(BWjT}AaI}50aRB2(xNs|VXTlmv+D94#8BapOt+1SKt2eYK z7=0NxLd-4Koi49WYgZe+7zrWbmRU~a)oa@2Mi)jz2)U)YQ(^V8cCk^&AOdlv6t+PLY=teb z88*R2*Z}Kc9jt{luo_muN>~BQVHqriCGa{dhDGoiECdr6p%@mxe3%DCPzVOlLjlZ% ze3%1yFdJsUOwhp$$b}rphUt(6(;yRGg;(HZcnPM$i!cR9m<$;(3DRLAq`?Fj5944g z0K5QWU^I+^=V2tI!Uz}+!ypBQ!gDYLk|7BaVK5AWfuMy1h=(|cg&5F4G(b8Wj*X>a$yhv=jiF-57(9m6P#RK$YuIQinvBMy*(fTC zjKZVXNGg(y#3R`VDuRr_BiL{%oD9dq*)S@M48z0N0n`9;06u^Xr9#P2Jd_QgLdXz2 zgzZoDC;Q|5Sv935)wr4srh>^}JeciA^&|V?{n#KXhz!Dm*gz_f48#N304jhCzynx+ z%AfSd{n@@$U$QUWmnA5IByfWDqx?ue+>cdJDpG~3*gjMrvJc*e^`(4CU)+~fQc6;Z zD_I4lAQiZR#VMS`ah#P?a#D`VS&YI+498d>%7^s9eOPbGoAk!LSue_q^uoPZ86_iS zxQz9rJV{U7la*3ZQi@Ai56Xk|z&%)Z%AIt_-B~xvjda7^SXauGbj4j+7s`cn!ChEq z%9(V=ommMbAtktkb)uX|C)|k@Q({t#i&;m?k#xizSrH{7MYxD{pd3gC+<_HRLQ;qe zSpg*=1-O7k6e1Ch*u9r)Z(16QZl)9eE%r z@ft1J>^+77+hlsxZ0f9mNg4O*)LI;F}M?^+NYhq&K;uEw32Mtb4 zN*?mu(3D}rN2HE?e$?nOFTmJw<0qs|OrMl7nVjeQ4~~3z^rMeIId;6_)6YKtf;n;W%TuS%oUJ@pb^gM| zOP8y!)YM+RcD?S#&0Fm4J9oeO`kQaRt8Zv*YHn$@eE+W>+J5}$-&}h~XV<;|+<(w* zefaY)zy9_J2?Rn1k)zm2;_Tw;=I$Z&lzDmkU~*ic^zEba`v*8c_)m9$!#~6U4)#O% ziq?Z(Z!YO1Z+U&#?sWXt)S#qWQx6}W^HTY-sh{K>%h#$7nCBh8Gh=uD@jKP>`g=70 zd|dg%ZQ)!;+sjA9ANc(-j;nc3l!OT}(UY^kzkMnD`$>!u*p>5lHaI+fJK3iEt`QXc zxEiT_*fTNlOiVlz6VJrNGcoZ@Ogs}4{~s|S!lqz)Y~$w5lt&UK@7>R*32a(}*AR$T z5s23jh*uJb*Aj?V6NuLn2s!iLvHaI|**ElGoUj~h{lS$ho^1T$-i<#_fG*gO_|0Fm z#4~2n0!hH|+%?X&o6BMcq(=|)4{q?Cq;#-BiLeB09JaIi&5#672vr%2pDXl)M(;gS&}JY@m0#WEpM?4L3= z#gCt^lnIpz5nn+3<9mK2CRECMnr$XOzLW|{oo!YAdaB#T$2RNF&3_N}WVFqHPlMPj zQ!1Q#4C?7o$tBoEg6DyhDtQ@!J(Ts7Z$x9`dW!ory+bM|U^@go?KKmz*ZHDSesYLy zp+%$I^;xs>3+84PWzWpb&%)gJ)?lP?%6uX9#mreb8NWSx^vH=10N>`BnVy=KkveX8 zYUa3%vBOd)X8gi;O6?O|Gbg7&Mr!5@qsNU1CgLC3dQ^wJC9=l`@w0_In9a6ep~A5! zyPz;PLx?apzUDA5OBamdMJg3PR9+)hs+4ooNX3&2mhcXI!US{U`{X|kk#8i_k|cXH z?4gmPkjJsH*La`sikM#rCGl|gGcdWlIvtBdHb{{j@-K$ivG*^QKa{DW@ZQt`B7?Ca#n%LM8h(dy07Dtmg^?=~}d4@+TU`$ZUZmN=;7+7w1eA%ZHM0BDDM=gYbL7irl}H8$deRT zh-@kUS43OBBKZ%KF+zSO1B*lQed~pMLZ~J3%5)!Ibd1qw6&mK}&Wp$|^k0x$STw6( zuD>QylMos0A2NOZtT{RUanUi^u>&>H*|FM~oR~og(XqLjff}t=la-x2eMW9}wpKGe YcSeFHR5^#w#6NvdIRE3H!2jp*U!0+s?*IS* literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-1.parquet b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/bucket-0/data-f31d04c3-09a9-4c6e-b16a-9081d4a3faa7-1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4403ca09ddd41c2ef8cac88ae8ce69987c5ad694 GIT binary patch literal 6570 zcmeHMd7RYNeb>xr?t9)yZ)P<+Z)WGdk3DcwitWUk5FgY*USz7K3WhZ%F@}KcUZLwc z&f%nSX_Q8p3Rba84I#vF9ZW2D<5HKpgb-L;fDponU}83eVmEQ!tdF=FW^HJld_MU* zMmuk%_j~Vsf9K5nzMr}A>cuSrBX%z!&LZMFk9Y2MCWz^vDRB`l3@bj~coSQA{gs&K zwT&nenrVOAf?c>O*Oq>g&)Q$NylnpQBlHLuQHJ?pdKe5VL;Mgu1csDBevlpngG!rk z({0dJT6~LcftJ$bn{*R2l?LCS8=#@o`8r((b*0AF=o+XgRlZ7BK~<^n6}kc{N|`Uy zWl&a1e2FfBl2YW0bP*Jl0$-pDprGXWJe>!5CCBIJ9LOnIK1*joR>|-gIs-Dw06#zv zfB_}Vr|C3ED=9ujr$90HyTteRLn_ zQ{sG_j)S-o<70FT#1zOw8Um=0JV}#)R0y7+2|y@OK1xSHQ~^Ao0RT#bkI)eiQNnzf z4uh}~;zM)@gp?p3q=O)+1o!|Q00G6%`)NP$D?Z*w`+!gJ@?P2tyo!hS&>rAX+`OB1 z1GnPhU9=0h6esVboxrI$cn9qO4#m#fX*;khIFHjfz!e*Bqiw*ZSa~aL1y;qvTWAZg zC>W2?7{HVpMh;)296Yr0)Zm+ECMV9#Zk?Q%nO!+KaeB5eIdO9K%;t&Xvs*V$9GzXc zdE(G)Ve`cF?3qmy`)9Xqn%FzLa?`|~*}^6`tPTl7%n%$>2Zcdq5Duzsq0O{mTWtv~ zrUhGSQ)n_x*i;)rgK5BqS{Ld}9oE&FP-AMardEY2Q-xKvB2<_Ptf*z7%#>kSEeR#2 z1WRgBC^AJ@R0~3ZDZql77xGLV=GB~#V{$O3W`!)1g;_NtWS9)hr~|?PGXMwFw2)@f zFs-J96qAA}6$ywz5UEKa$s}P??HBr)e%P-jgang-36&BkhJuvZC-gCWuuqK(aV8Gq zYD|bRF&I;!02v6ON(v-HLQ*9Jf*~NGMujL7g;5mqR z=u(}6lW{_)>JS`^13FZ@U}x;muHpjD;1E}Bf{n33n`#xTj1^i{i(p|a(4t}j#$XUr zKX&)wQT5>AGpE*XKHoXlIop}(yx2M2In_DYIng=ZIo3JaInp`Qd9E|vIndeP+1J_I zd8V_c^K@r-=c&$A=gH0!on3BmkR7B3^|sh%+f-X`i7mE8we+UgWSdk|Z-@=HK{fQc zSZC{0U9X8Xwno+Ts#s;KR8_Bt6}CcE^s-oH%T!q}i6yo~mGq)mWQ$Z$FNg)UKo#`7 zm}m1;UeAd+Hb>?3te9o9R94T388$;@^Z{{z9iRsEw3ueoR9a7oDKCeUwl4ieA=Bd3BHIVLg;bcZ+V;O}TZK=we-zOLvM+)=4>ahv;A( zltZ_RcGgbWbzH<*oWgaRXk%@ZO}C0x)=F7*i)djjltsrxjKwHSU%l(_hxCI-R=)7v z@1K{?$!Fym`9=A(d`dnkpOBBs$K<2(5&4k(oIEWbkoU{`7E1{B6l!{yt6^(*a;0mZ<JE0xd zj%i1=BibSDIc-`ypzYW8X?wM2v_0C>+HUPBZAyDmdqUf#?bIIA9@QSvc4*tRZQ8@y z7VRPJ8``8BGr!->FaLWs5C=lUk57Dg2mbDM;CKqV>B<=!_S3UJJNKXGUwZk!e*TMB z{`=Mc`Q_|uufOr;uinCNi`9nP9Zr|qJNPIn$Zt^c;QE`z3%!O7X87EA6-n}G`57f zdFdarfAq0G=03jcmgOtPZ(VuYs!x3K_SJW+S^KGV>+ihl?mzj{Kl}5)`1FRqoVe$& z?)}VtpZ(nDzi|H-zx3B%e&BC5{_R)zum0WFgs*r0Ufi^KQu@Y&5B9Ozbe0~qSY z!$;Jk+A;n3^CygxKR)%s>7TrKX682q!2VtjfaAR$0O#*m0Gz$9|1)Q{MNS^HoZ@Ve zg{fcK7Ps%eFM0!+9_2?DKD#3Py=NUSf5P7V0KrZ6kRo1s>A*Mf`Gl3g-zE~N9!&2u z#z7>BSEQ}?_XtP#$UB51X6~B~_O@MSCYPDXWoB}jnOtTjmzl|BX7V0qCOB~;aU1dc zwCM_Ajgxa?mr21cT=AIe;9P@t$#PeH_f+?Gsrc7}y}i2|rAv2r9Wg<= ze$0szq>GqLnMPpo1o?1^xHmlae7;32ntPmTU)(Z76Zcu>?H7+`XWy<kdV4JcKM~{AVY;@5FN4Z5~ zH+*1p@z@*YVz76>%PqP3`ms^&x{oZnCYKz3t-F%g0bJn=-k7JYCRo?8oHb@&zkJQQ z@i8k#_{^E@&Sfie*f5?*m|Mx|U_24OJ&z?!<8m%DdT(1o_{=r(l?CKvD({-#OGPg) zBv!nTnBL5ciOHT#Mc8~3?&Qq`Nlq;x>R1;N??JxHkskKW4dlJy{KKjF7n2!#C&Q+a zcT#*&`hS(ZdNUCt&n_VrTu^4(kGtZrO9Ru>vx5ApC4at^-_}a>YNPkR>xLNttT)5r z)oxLnZkWDwZ(VO5SZaE+jhXXiEM=rt*<1R;hE;~KQEpGBF z*8B{eKepJkwOg4NZM``AwYE-m!SB0y&Q@Q%2i{F~5BhU9a@vyLHZGNNqE6x7`YCx98c zK}tc;e?UA|^jOb=Cr?ESejpz8AP9vXdMLGZ-hL#h^`J{O@4fl>&CG9R-^B2wiGW?0 zz@4q1-y#|^I?5;mBwn5W1tS*$-bVp8>Fod@osJn?|FAQB#PR{aJF*gw@DPviV1%aQ z*4D;zmPH9!0Ia(fJ}5&Czb`*p?i8rPqZz-8%tBt||43590sQgh#S<3~Fw(-S14sfg zB*KU3^{`Wz$Gwf$pF&<)Z4RFU2g)4oI!?Wl)@pIVvFt{zPTp#_q$*65OpD+NC6)VP zIbcLpIY2 zErcasy2@~gwkkg*f-H?3uNNz`wK;3Pe8-tBVvMA5P@>jY?xt0-3;PEL2T|%W$-1l? zx$A{o{&LRB7p`5(O&0b@dBSrzW=#!`7jo9v)%=zC>A_v0(O#aEyWL{QTOu$~S6ouU z^W|E@DM)}ZI)xUBx8pF#G>uF-Q#G|}fJV8+W#TSK1Y?}>AwFfuS+5ITOE@Ilo_Hss zf+m=PK$f7^qu9eAhOrj}9`o>y6yXAobK98CwtA>E8%qJ2ooz`E2YM?4t%q^Q4Md4E zSykEzabw)#k2{PN$#~PDc@yFNiwdZ4@J^tfb-xheMtoXcAz!1&`F2oJJMOU7D=@WdFlQf!?L*S;bX-bR2I^YKJaT{N378LKRt zf<$Y66(%OS4TeS$_){4*z8Ap^@Q)$R44Ln<9=c3M>&3>sMQ7pMY(u~AH0CR{Io<3v zQ@u$&QCh52ZG9lwSMJZ4$#Q?X&+faBO7=Tu#!RQpV%aIpIOTHMEIBhN^Nd=h5&Po@ LVY(?M=$8Bq{gK@p literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-1e092fe2-2c76-4682-bfec-50ebecd3957a-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-1e092fe2-2c76-4682-bfec-50ebecd3957a-0 new file mode 100644 index 0000000000000000000000000000000000000000..025c6a081174891ce38b2f0d25ee553584781361 GIT binary patch literal 4096 zcmZ9PRk*NSQHJ;GUN5#-Ad1+EjfH@eh@fI8Vs{tD?(XjH?(XjH?(XjG{^7#|=HQt* zm^r$yDeUrU-lzuzBVQ+6mO8%jR6mP$d?`l3wvM-pdS5)ERWYuo#b3{ZQ2T5LMKNv{ z-nLfs>SD183Q@EE$xKnH<3+m-75{n+>YYpLEwY_Qp0CgOquJ%RAXR7LXuZtO%_I-h z{oEC}!*jH6Tluc3=CP=dyX$lw2$#(_-Ktpr^TGUBdGcKp4gEnpzGsQfDce-bXq(Ne zR5)oy)gntrs;k!qgS+Ft0drtK&#G~LuyHPT#APRC(CN;lIumtMdD-18x3qsHg9>N{C4#%N7TY|H20?t5T*attM z3si(k>>gS{Q>X?Fpf;3c-`n6M*f8s3pUh8Pm_4&%K3wKZnE}&gYRsoZk;yP|=C?wO zmwC`<`g1#aNl)m%$#MF0i>}fI`rp!Yj1JOnAj$F2cj`oKsQ|E`##E1r0ufG=s!(|< zMWui!6`;{=5BczL*!3I)Ba>xUGPZEhBe&hvvp(}iV*KiNO@C}Py2IN97!1!&cZ2C*d;Oh3D`Q zdZSPjk1|m)szvQ+5KW_1w2v;)GxEjZI1y*#Qe2Na@i3mn>-Z2~V^`u&B1tmICFP`% zbdymsPd3Rhxh3v2kVex~nolcfGwr40bdhe;Q+iK5Sul%b>8y}dvsTv6CfPDOXOGOA zhw^xy$%}a{Z|8%2ny>PGe#xJ?uLu{3B3qP-deJF{#jIEthvHheN`DzClVz?fmyNPp zj>>ttDUao?bXS2YTBWLdRjHa)uNqg2YFnMEd*!Kvb*xU;g}Pd|>V7?`m-Vhb*N@uU zgqnDhX^Ks)X*Yvr+N_#=b7`K9uMM|}HrtlkdfRD-?W|q5hxXdKI{r?ilkDU=;v^;A*=?*u?$wkieL?EV*{+mnPM66$gZ$GcERG{ z6Z7F=JOn22EMCIvxEJi;H+G26@CUHQ5BL>-;b*{4M2IA@19F6$QzjZjmskQLVoq#` z32-F7=0>>50NDouoG6(h^JELCkWI2jj>#&pAh+a+EC5l?o%B#aDn_M&G*zIgR19cQ zeQH9bI7=!B?5H#Kpge$=4$*NsL;pvSuF-i;oBq22J*8Llp8k^y{iKzT`JFJ6U@Dv} zQ(}I(&UBa|^U)czW)94i`5E|PnjAkHVShNu=GZdZf$Go@nn7!*!5*LzbcJ4!AI?G% zI0@(A?<&Iyr~!B3E<1uT{!(9nWbw88td&Lzf8i?5fhUcJ$F?3_vS#O+AM$nn2xj># zdWO;DnFW$tJIuekllMCH+#8m1U%m>nc_I&m;d~ekLU(eD`{_OLMaOs(&trevZC&lP znWWS3XAjM~nKkA3WA7X`%k-3Vno(SD5>cs%rIpUF?9ybLZ4wO%H|;PN7t-H7r|Hz! zgrjGDsrU7tmfLOeNcZ8Yp4Njp-$}-QziHdGH(MmNx>)N>Jx*%jkNjsS>m~6zRD0`4 z>gGtq|T9rS}WVtpLRI5Uju3}X+O$WiMn9W;H746(hS6mDW z!Koazwd@jZ%SAaZt3j`9mX)#$oDGyGDnyf50Ul0|C_`gn zK}>-jyd{p%iMSIUasUL$7?~#9K#)@)Z%~zNk$tiTOvokKWOrl{IFoMpL3*heCq!j{ zIF*AkRFR4UH7WqNsR1>mLcogJQ)$kH@&Xa)Nkw5F9i|iX=S`L_(RKPCI&=yi(gn_p zUekYeps)0c{yjgFha*gq$uYlLW|B~q(_p&H$H(x9slan)!yK9S-xxQ$fO~9!-NGl> z58Yu85=36$kH(NRQb3|?6}f^fWX$%F3333N@DgcpcE}k?u@7VoeybP7AvBI+`ld5B zQKjwF*{~kE+hsDWZ}~Wmm8p1E-1E`Kf=HDgzE34Xtr zZkkK#Z`#2+EF_h9mqeO*)5-F2vMH3G*{8Y2(-}42+X>6fIr)WdbIeR!Fe&GGB6Q`?U|9~sMNv)a*{&Sr#b6(t%W2Sv zy7@YD$Cctyrjtfq%4(^%$^_eD(T1w8Zx*r6sV#-ABwo!jPm+tud9=t>skYf3vSM{A z;(;%$Rrw;9XERrnN&R`diX}D>AEQB)%tOJnx)-CUpG5Li6)3`CB8x=(Y7pGxZF{MP zSv>Uy>CU*#x6jH`4$@HSt55MPEJq|7Pm*x`Ux7B9CF*{s*ZQM&dI+=iHV&4hdJ*@E zN8&DgS-tMX|A^Te787;?NAWxUAi8uN{qY>#06pxJ%z;~|iI3<3GRN*r8CxM9_=U#F z5jdy&#G3kx2JKHxc+paRSh^8x?e=2Y-FXTj`|HWFouNQ>Qo zUF?Z;&;yz#vQ&ympk46S3Ty!`k}YD)B!E0qC9c#3iNW7j<3yn=u!Jkj2`^Doq{p=2 zw;#bX;m4YcAIxES{1Zufjb_msSf^@a7`$16w@J z_SpgHAv)9n7(*$1fmhiXkl{>8FXs%GK{w&ckccA#P5^h~FQ5lq$Qjkc15gOT{0)m%yUD7>Hi{>$q8IgxNihrN*|PAZ-p;Pb zB%|zH41-5;if*B|{Dx2&FFQdu%ap~WSk}sVkPF*osB@17Wh1Mn(=s2e%2M#QeOXKT z@uf_pIBNGwM{~U=$9kL8lzfs$iVV*3}?6 zRKDOCUMp8!Z6le#{+~=IQYY(a5{q-SKP%V&oQfKCI4z~!+Lev!zg#8rdQ&&TbUY7_ zb+I#PZ}lHuQ+E?+T5+`b^+UQ(QcWSwH=D51tkY)mi@oNOjGJy)>nxfN&eCnePE9o~ zwD;!wJ?&2{<3RLGhUqRo$Jz7|2is^K$ouWNP38V@+2-?cRLPrpB&6Q%z*K$p=ld4bPR6m!#KumawwA#$V! z#1r-seqcidaSs%NQ&=00;YZ-f`p7wzL__R|iU4Um0EUqZ+oj^{lW0&U@D3L60i58J zsSVIUa#S8m0wJP`Cn+x+p(aoZ@>4nBMHYz`UWMG?2)Ga)da38ae-$voJHcK8iw6CJVvw3rOL zK|fh1`^=QBGDUa`7TIqsFbAZBOqeuNV;9&JnWMiEV~*$t zV@jMmJtc1FXEX4LZh}4R%>2NXULy&zf#%Q!{Z|F(!PMaQjp-P)VpCuQbz=|Q%XWbt zjX(Ly0)nr6RhOXG&+qN8S8mqpb@krYJ<{|#zdrHGYDT)ad``Q#oER6E&pR$I>NOX* vyxZq}pZ!8La;bdBTSQ%4Qr`cZ`Gw%ir0%uoEwf*A|F3U-njinn2S5FPf=A^^ literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-360cfb9c-6646-44cb-aec5-d156a379b4d2-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-360cfb9c-6646-44cb-aec5-d156a379b4d2-0 new file mode 100644 index 0000000000000000000000000000000000000000..77fc5c5306a186e43f9ff59158d372c80a2bfdd4 GIT binary patch literal 4083 zcmZA4Q?&QZbqDY>_WbHLRcb4>k@79mHZ@bbO{$bRwr%_Tk8RtwZCgEW`dno$X6-dI zcb~QP>@RkETfV3dG&*6(2R*`bxC@uzBppAUhp0)f+Twiwn2pT zv(M(z?3-2dW#!Ja88q#t))bp8lVHNk*XKDt=24&PUA?R)b-!-aA4+kmb&<}{aXLhM z>3emmekjV>R*Py}^{Qr7sR~q@`sn~CMg=Jkbt{kMrkt0f@>gDMw``QbDM z#qWq~aVXZstQZ!ZqF$7WKkasMM3RUQe&U&5@_oL_|Nh!e^FiLuYk4ux;t4#A|NEi! z;g9T`jXJw*nN6~O<^k4iE30NjEQ7_d5az}1X%LvTr*xYx(s9~Ln`tF2pfOv1J0wWT-*C*cU} zho8|U+DFacb63$c8bs}=78Ro`lz@5x25%TXZy0X!^S=BWzVnBF<|DuRXaDd?PY?`Z zK{_Y|RiFj*feEk#cEA~U0NyYZ#=}hLf;F%W4!|k60{7qre1g6x93`S`RD$YI2O2^% zXbl~pEA)c=aU@R0xws5B;4VCZ=kNwT!Z+BR1d?cyO7cksX(Byjj4Y5XazgHiCk>{t zG@TaGD%wK(=mcG&JM@e`P;VB>;#nqhu^QIK2G|r^VHfO)`SNg{$g_C~uj3tjh|lmf ze!#Ez3-=e1B3a~$GSMKq#E6&^8{$aZ2zMDMqh+ehmld)}_Q)~0Ah+a+ypx_PSjDPz zRj8^|i|SJoYDw*=Gxeanb*PTlncAgmbekT~Q+h@3=?nd&eNDJYG})%a)R_)5WM<5o zIWSk|#rWGun{0D!nQgFLcErxv4SQs7th*EFL_4WYzEj~eIX%uE!|E0GS~&PsNAO5 zW+%$Ff_0RH0_>}~HiziYteaW$!D}>ZI!&W}MVGMNR63<5#{}Rc6Jh+!K8(WO_^emq zrB1>7dR6D)XT1xg%W)2>)!$XD8*!FS&|%s~cjE~3sLoZIGm59-uKJN>HL2!t zziL(0s@rbjL0F_R)G=;{aq4G7)Gdy2yi^gmmmZ`Poyu*wD1*qj?3KUTEMtfts+0vX z$VnrUAWg=|AXz{>I`8vO~r(~A@<1knEq?6b4Qtm~wU=B~>5&ZK*s2~52XLfAkXac-s zAP!*k&SHa~*i0G@*VqEqGi(SBu>loNU9?TLkPcR(ra&F@bQWxnsG(iR8;UzJPc#!;&=cI^?9n+M>rBx;5(-vmsCoh=&;>3) zz8yhV@P*ysJ>ZX4KpRN4k$evQoVThFUl45{yXGq7& z4qe8BrMsxWp0Z9S!a`D_>)>~1>l@yLbJ3kh)VVxd?+{;oA_hPY{{1I)#nPP%RYi{Y zp4t*)*qbhhJG-LB#E(uXjNe(bI#DPE<)KPjjv7%nY6qie9&Msybc;N25N@_HxE7@0 z0$hokaW5Xni+CFsgHzmV@39vNA#o&w6p?C@1zJfznIz+One38t@<@DW0tllCG>aC~ zTDoZ4=^&k^VPKW+(@ScesUM4ANo?EZFdtCL>RBfnX0vRa9kTzrX0I%OpV}y%!t;1J zZ{#2E=A(R`Z}MY)Z*RGW2oir8Bho~Hs1(hjSBwiUXHopYwm22{!b^t8IGG`f8j7VHoA z$b?PScRZVZZ={}s;q<{~I{rErdJ2E`qViE6{;N55r3!5z4;Jp|KuzEQXHEV0l3gJ) zDptJEA+>~myhDw^Iu(sdK)M*BC29x9JJ~8xUC}9Wh6}|34Ob7?S3OBg-&v3-c+8UM zE%V?JbQT2h816^Y___5!1^k~Yc?xXi&!m?pfpLCG7Wt&z=KJK7H=}$0*Iwe$=D`rL zO5#L@m?lM{S`3m_(M~!+zxbm`Q4W_yE!h>0k8>`H$)oU*IUr0{q6GOHStLsq%SM<$ zYGu0&BZD%`>9^BzRqo4Q@FAD-S$0D|^@to)8RR!qpi_%$6E1ribI!~ADYTT%cY_}eCMs*Qxwe$L0H+2Tygva_; z7dvs-!wiEU6JtVfnn^?TAP5zhudXy+_!u^uUUQGe&7!GwPSLhGHTUKNUUnPZ!Vnv0 z7tuO61mox$WY||QOV{bLJ*3xknD(Mq8UX@W6!Qa}GytZsJXTN3*{*G5rL>z>+fg>p za_A=eAI#oa&=)yEd*Bn!wEoH^##p4fz&A8m?cqW+LvmFg$`obl!wo9YS;6kCOT~*3 zH3ozEoO-fTI3I4P&k7Ysstl}=Kz5@Bu(xp6-|r6sb+mrqU9=5HQcsks7htyYH_;KEyMOn;jvUtpsldy=@^cqbKZNLhT()rtx+OrSl%P0VdE9@TWEQ22{X3u_7*n zuTw(YVI6rAzOo9f*l_uq*<_ARi9{L8V^Oy3aB{&0Es=?2Ks3P)QYQy&hy3Gk5)FrB zo9F^FGTb?$Yw{O;$$_j9S8|A8{?5XW43lNfcKIlswy$y?GoZYiU)@sVtZw*>6^N6zWEIEZCJcBo?L;0y%yNWxVXM0FG$*S#E{brIUIUX!Q zM1dj{05-v>YDFHfmFLL69ft+5fQ6uW<>ySANi>Z+Wr%46uVhOp72^&3%Ly9~8UBL)Vip=?1l;9E|JFKBl)I zNMDLLlLDGqJ1&)vyqlhz7~QMCcHh=Jr|jD+VIJM2X?k7uqieD++U+8YkdGz~73gYI zPIH`7U5j_oIgep+yi(88Rgqze__d61+J%ozq0Kr21yC&BEZz@$fwTl)i9Vf)M&yL9 zLBG7mmvlCM5xS9lXW(=%d$CgKt2NIvKt=r6s^6!&++aW2WWq2}vO zXgVxpT_oO=Il=ITMAJx_X?EBW^Tm0C##}%48GRLoIBJKA8q*L%O)HeZeE-Pux|V zZIC5m%?7G}yTW}i+(xSvbb!aei%nHObw@LKAc!RSDqAGlP~HM7R4Scdg>VhektX#3 zQ$deP6rOBMZGmK`%b7w4c0tYH30Pu7Q5)J)Gd5gw$$WaEe((eg;XZb!>V&U|N5L$f zdun&kVfh9L);+WXKk+*ds~5lkx{=;|O{D7%alsEnp{@XRcnMbNTql;5kt%q`TJ(3k zQMiS?1-=iw4YxUOL)C}gK9J_O#`WGC(zJAQ`1Y>bJ0(J;?sP{E$@Tx fGrm3k_^SPM@fn-T`Tv8@ynH`gzW31&f8_rF-ZJAA literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-5561631d-80ef-45bc-8b87-3d1a2876816d-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-5561631d-80ef-45bc-8b87-3d1a2876816d-0 new file mode 100644 index 0000000000000000000000000000000000000000..07832286cce284be239917d0944a4aff7590990d GIT binary patch literal 33 jcmZQ%U|^7lbN+O}%7B4^5y%0N0zi_JL6{*lIyn~rP%8wL literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-806143e3-9271-4934-9ca5-636a815c46a0-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-806143e3-9271-4934-9ca5-636a815c46a0-0 new file mode 100644 index 0000000000000000000000000000000000000000..c9776973c9c01bca6a2a344ca6d39777fde2840d GIT binary patch literal 88 zcmZSaWnf@nVr1X|QVa|=Yj@hPfJGR1AtFj=j!Xngp%Fq*J};U$BarVI;uHV?Rn`V| literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-a353b7ca-671b-4423-bff5-fd2888db7536-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-a353b7ca-671b-4423-bff5-fd2888db7536-0 new file mode 100644 index 000000000..6b2aaa764 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/index/index-a353b7ca-671b-4423-bff5-fd2888db7536-0 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0 new file mode 100644 index 0000000000000000000000000000000000000000..0879442911e32b311130c5cbcb36580ba15714c2 GIT binary patch literal 1445 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~m&y4=-o1PA`mcol5x0wu+oqgZ~tR|jpc)hj28cV4md41x%{f3$JSWC=Z~&m+L*QOotm+D;btw>^|HLj dsxLj^FqdIwWoYPVNDyFp#FNk=qrimjWB{~c@vr~@ literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-c09d3b21-6083-4881-a6f6-b0ecc37dac22-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/index-manifest-c09d3b21-6083-4881-a6f6-b0ecc37dac22-0 new file mode 100644 index 0000000000000000000000000000000000000000..8c6909a73115c02a57e3f5d370f620b0245bf6e7 GIT binary patch literal 1589 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~m@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~m&7&t_*~t+ z5=)XVUVp^0hgoB*{$B=(Kl_@dT$ zAGiwy!83c0M%PyLZp{sTzDl`J=l_56+`z!ZE~YL0JN42`97U$bPP(19saNLWx|0IF ztM$x%u228Z!1Dj`|4D`ae*AC!FL2_(&xsixW;_NLK4lkPytygx_RY|BO?_*tgV(-V zrS$4bY-Y-n|34jHhgDB<;t*YXJ8SB0k!0=5#iu`?yejtI^)e?XM|&IR(}v29gc&LU o60B^F4;bwkIT;+LHB?^MsI4%A;|@ny;-S_fq7zOqDxrHA0QS!d6951J literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-425d391e-cf9c-4770-9241-09facc794f72-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-425d391e-cf9c-4770-9241-09facc794f72-0 new file mode 100644 index 0000000000000000000000000000000000000000..2c7c92e21c2bc4f6ac2f70791bb6d56aae76a932 GIT binary patch literal 2169 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8c3IuA*v~H0-nzg| z{p#8K`&s5NYi!m3%iy8Q#;}M}S-atACQlBZ-{a}7k`tS_K6o5Cz{&YuSbg_efo6s) z`(}5T6*CCvN#0*PRhMi+VFS&>HY7uBfKP; zU%38Td+==i$KBj}rC;2=^QUcj*5p1rnK^g(IwCxS%v3jc>UVUBO-@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8dIDZFu_))x%>1wU z^uek-N*q^MG`8yhW$@tTWJo%1tY45hi}_7_iR+cv@?WQ97X2)+vgHst?S5o!dHM8T zYvN8s6|QLV%4Ou zQ-zH8O)m&u1fq_v+VZ-AvW5++wXaw?|q-|d7p-c z&TOe}ztd62f!@6+-8mASsVc9%-gOSw}mc)W48AvO#kch8qT0_zJ zNefC$7GbDRAL|W;Ny}_DOb{p8030wH2X=jeSbrr5Vqj?oV@!EAB>=1P8igBZp?24c z5n*T&muRpuiVPEkE{91r2oj^B2#}xh_Tprj%OEiXg_l$(=FfqqTO=ECf*3!{jfx8J zsFm4B)@nwr31c3e(xBdWOLQl&`jkfL|1X95ax^>0=pExUoXkT7L0Fc_MZlD`}~dQV7^D5iTb#&4f3 zAE4zJx^aqTG?gvBD)d%a;ywyTkMxfQL#YzU)@%Qd(PWH|?ifRMNyIog=GdJRZ~yqV z^ZxJekM=6{=a;MIsyZI*|K;$*YR61%OZ&v(&kv`!H%~V-HXq0=%pY|>{n&Q=)4RtX zUR`^C;dFp|eF@CEYW{q=xi)#K@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8Dx>P}{5Ywa6#Z$} zSDEM2Z}ZJz*VwB6m%&4qkHPJlk7YpiEWU;?pP8+!mtA*BG2MC~k!HXC+5zszT3c$T z8NU8DEq48@?%C~em&&4dAMf%D3*c>AmX_f;NyM7LhQW@3VT$8vj_sy`7ysygSp7== z+#mS`j7Ba`T9`Puact=2QC*|y5J=dzc}hw{ndCyZmFCV6Beech}LWH zKfV6iRJ1!d(`Lid%vtL<-VD9UEqi$Bf->p7-@?q)XDZJwa6Ip!R<9*Dz1URzd6wR} zt1)VAY6lmkt4y}n^ZR&zlF|S5Vj?fZ1o93Zbne{IA*J?Aa^q_@$*nqDvzCh74f5I8 zyD0hYBF>M;G*etE)c)r?xJewWXZ$bA*T?!OM2FE;!9iKja~rUAbGe{UY2J^!Mog}( z9qe`6j~H~x=BWtW?0w_H{#M;#nV5&ciRTxSxId=U&AfZTs)O5(sijBu#{}84M!GK= z*xvk2*j%AJVdcRI79FY<2Zh%xW-;zqEP1iSC(@&#qutYDcZb~+MppHd@}`EgKkA>~ eaz47wFV!DzT)gXsXS#>V!2^2DGOA4I5exu)X-ED5 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-dc356a38-74d8-41b5-a7b8-270860bf71ad-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-dc356a38-74d8-41b5-a7b8-270860bf71ad-0 new file mode 100644 index 0000000000000000000000000000000000000000..2b0ba83c66765296d84378375fe9d7574daddd22 GIT binary patch literal 2137 zcmdT_QAiVE9N*@oCxc2fq(V;=#kmTCa`fQ2ed@8>rnfsaZC$tBIZyAlb$4b<1_l;J z6w89B=N=+@Y(yf3J%mubMnMrp)I&t|QW%JRciZmH&22sO(%ZK0_y2wW|L^<#e+z^s zt@b+Rv=n#BkqJ&@9_^h)lw^WKT!_R);u4EEkQgHteU}qVMEx-?MV$=%Ji$2=3@^rG zPK_+e#W;ykIJJD7=LA+pTw}9&CVB`EK`h{hhq=4>n1ZAn2EdE?J*7_vP=dtBO$6bj z!pWN`dMLLSkflk;0Eyys78NWF>&0-OpH|R@rgP{mMpjQ~Kk75c5LuDoHLs~2veh1%T^ zic`QKF416Rl%K-!tQ-c}7!sQn1O)miXD>#Qm?RW~S2$61V(uK#bPJ_I4us{#nQ1{m zTxw-1lro!9YeJtVOQ}V(*Iuy_2q(KkOF=;bTj4a0fgbPVi!V-{XIJ9 z?gu`U#!QP}#E63-fEza!#s(N62~=A`!wmXM6_`L@Kl4+&34p4>LX((jQ}ce&skFWv z&^}QdVcJa$~lFgg|MYxUB(aTkZ%N3h;18yO1$ew4=H7g~5Id=G-fbheZ9zeH#te`mGDcCoR2ZmOZGRT_L|UpaXXPrI8ghpE-{*ZTC6 zh31+|r{pV*mdKgsFIca2w(^%bS>=AGHl@VUuFpR4u3)s64FZ1nhArmJc0 z@sGQ|lkcj&eeUZ%Uvs$qnwas-k5Ri@0Dj`e{LA}wY}-eB{dLbuyHEbL@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhUXTG3b$0ecM{6;4vC!p{SVVBMvblde;GVdnHZ8<1f&nB%KrcVe}5rEY~4xS%U9G- zJE`2{R(#(YJ56NU*|g0!(~=(xv78frr@Cs@+2Rt-dV%+|R>y0!CV7-QY`W%f#c>J~ N!+{6J3?k_E006WXbf^FT literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-1 new file mode 100644 index 0000000000000000000000000000000000000000..eab85c7ee6cd2b6faff0a8f53484ea4bbf6b31ce GIT binary patch literal 1112 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhRb*5ezm4=NS52U?f8tmJxomN7&W%)|7GwfWnySz5tQ6u&K|+ytlBhzEl@apQslKf z`G@+sJN>tbm~Ak4Sz`6p#K=lL@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhBWPeYBDQhzXh%;jS4+`=^oo5CXKE7e;GWEvM_w=R8ZQW`jY1`!y>C<&o%`Yfx~Li zb~!Iv?i6`@o>-Xv*y?tIuw*&&q$!M^>qG=T_{`1a4m_kg@7k)@RzHsv&F5*ol3}LM zX~@7{@vb>jeM{fB%8BcRo(XE#I%|ebzc+Pv(GkIPF3ulG(VxmzANw(xRlfZ5*|qgD oW;4Vpo0}GM3an^k5N2m!THf-ofXk&`aoe2J3Xi%OMbO<00O4Mfxc~qF literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-1 new file mode 100644 index 0000000000000000000000000000000000000000..f5048a3379a71d84d00ce35f6ab74be77e4345ae GIT binary patch literal 1117 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhMd>jQ=Z-|Nm%#Q=Ck{UHyTVk7&W%)|7Gx)%fzsRg(WUGF)uSMwYWsr(mdI~BE>RM z*D}@2MAsz6$W+%N)!1Cu*x1w{)i^aN(I_!d*WiZ&6N3a0GBE%V5YC&!2*M!Vfd|G6 IBIwou0E16wxBvhE literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-0 new file mode 100644 index 0000000000000000000000000000000000000000..ba35d107eb7349a9adfd048815365fe411158201 GIT binary patch literal 1006 zcmbVL%}T>C99QtXgNP3hucIdqvRiB;7W*MoLIr>!v6{SlU%QG?5G^Amv>6Cgaks30Py9V|gwq4yVB4EdbZmIf5 zc^KD;E9x_19#!&={zUTE5&dH>xCKN1| zlbY{?Sx0{EHvwgRGL8gjNoMZn)nPg6DKa6}wO`7Wk&Zp}47m%?!bJ|hW>L8eGU&$K z!q-3@WNc=;DYbBI+ZWYJ7HPjaU2b$K?*GVszBhYyK}uP$dz%T8P{QdJeZhC{4+kHI LN6(G>$G4|XJn%&@ literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-1 new file mode 100644 index 0000000000000000000000000000000000000000..557e57b7953d03a2d692dd36175eccb37fe90c35 GIT binary patch literal 1110 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG z2E96m@4Rz+PGWGd0zz{#|VQDdwAUj~m@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhPSB#EcPN^_1-hqhg>+_E5q`GQDdwAUj~o4%nXm(BosEN%kmr!P;``F<#5vSa7qz7 zT)6Y8SmyVI?{}Oto1n&=#&GnI1mojWbhs^MOlMkIyIm@^Cuz>R! z7OCE C%7Kyq literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-1 new file mode 100644 index 0000000000000000000000000000000000000000..233f82c1806c895d3b1adc51039a74fb056be93b GIT binary patch literal 1117 zcmbVLPm9w)98F6vUOdPmUY(PNrgrUiQwoBynPr1XDos+wr4EzKw3{V=%4AlkOHTSR z6g+tFEC?Pw`Y{A=d-u5DOs#Ef7hTI_AT#sc@6CI^q#vAAYB%uLA6*#yX$kP;O>y(!k>Vm^P+3O0u*uFE*hmWtdY*A4fDMIp`jp{jP+D zaNwGKM;tD+qLhQ8z5*PW7DSG2!xbdwEJ`PvDA#;eLdDq{#_?^W8wNCxtm&@mU5YfI z1-&a^AL|3p^n?ZEdb+o&`j7As+9X~Qk8L_=s9S#dy1|Uo{15lG{`3kc$G0rh>-$dW zmPa5xjbkA4Uc`?X!!r?0#AzZswKxa!j7G(%4hS9N)0l&mMXIO8xp0DK^onfD|9V5gw}Sl+fs$%xrC@W#uWqk}M>^`>ZqQb-7If5mTC=m?4940nCTjEJ?W%H1jH)6YG2XqY$SGG}zO|H{ Gb-AB^;$zPM literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-0 new file mode 100644 index 0000000000000000000000000000000000000000..f751ffc0248d6481dcd8276690c478dec5615a22 GIT binary patch literal 1219 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhDNV1)_V@#da7&Zx-;*s)CZ0hW{s`-e;GW^vN9~{QCHcZ`m$Zsvti=8t)?mLPArCR z_I_`i?0!Ob`;=;9+vv^!M+QZsa@Qtb^f?cC%4I}8q(*whfU{pV-;f@QRo;9uVYY;7E}?{D!5E W@9;yBj(<)$meU=Nh%;-TdkO#)m8A9n literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/manifest/manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-1 new file mode 100644 index 0000000000000000000000000000000000000000..469fa0de9b16eb3c98dbb6728685d7e73d6a4596 GIT binary patch literal 1117 zcmbVLO>fgM7*4AS?ZhNBO}kRC&V2IyL4@6Cm2<7h~wwO&+{f7@1#`O#Ah_EW&SaV@wc1jfQ?54 zv;ZZ3N&`>@367W6*$DgMWlD!N%%6`*ZG^)(O==UdAxQ|uj0pN5BvFuo*3tQ;66(RO zt#>TGIrRz?2J-40u&)~sS*i)=kc`nV87`x2{aFDOr3)C#HIb@mP(wn~p6gwS)Sv;K zYha(KT}O9#1Y|p^Ggtjbc?c~Nuc*h=E!0yDw|HD{%t-b}dQ&~T2Fh{`1GPJ@RXF7l zNKT^&@V=+@V@mOuR}(T4i%CuQ!K9;p-mijy4De~hKua+5eqJ1w-9CV(igfKS<%&oT z-S%VX%s?|2G5jTq!p9(juFU1W2JAs&HQQCG+_l?zTC8N2_IIa?jV{FfKXO0cnEZ8q z8W8{1i;UqUz%-EVK6o+B#@d#&C6y(9^Xu(Jsr>$;BvxO( K?noPpcE12%FJCtR literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 new file mode 100644 index 000000000..072417e06 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/schema/schema-0 @@ -0,0 +1,32 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "score", + "type" : "INT" + }, { + "id" : 2, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ "id" ], + "options" : { + "bucket" : "1", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "parquet", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866466351 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST new file mode 100644 index 000000000..7813681f5 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/LATEST @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 new file mode 100644 index 000000000..65ae67734 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "uuid" : "67d8f6d2-d970-4968-9e30-0065cad82aaa", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-b6b5269a-5837-44e2-94c5-5033b77ca6de-1", + "deltaManifestListSize" : 1110, + "commitUser" : "d042b1e3-b731-4f72-ad18-29291dfd2e63", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866467778, + "totalRecordCount" : 2000, + "deltaRecordCount" : 2000, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 new file mode 100644 index 000000000..a2257b73e --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "2cf4b059-0a02-499a-af85-db54b6af0d2d", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-0", + "baseManifestListSize" : 1110, + "deltaManifestList" : "manifest-list-56d505a4-531c-494d-a72f-4adf49a50905-1", + "deltaManifestListSize" : 1112, + "indexManifest" : "index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0", + "commitUser" : "9e2ab339-ce47-48af-83e0-2ed698c964c3", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468252, + "totalRecordCount" : 2000, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 new file mode 100644 index 000000000..79b8ead14 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-3 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "2141b3d7-41f2-4cc6-8378-bf7022b59254", + "id" : 3, + "schemaId" : 0, + "baseManifestList" : "manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-0", + "baseManifestListSize" : 1149, + "deltaManifestList" : "manifest-list-d5365195-730d-4f66-844c-7ef48b6df2a6-1", + "deltaManifestListSize" : 1117, + "indexManifest" : "index-manifest-22adae88-c6c0-4120-9d67-4961b05d16a6-0", + "commitUser" : "61bb613e-40c7-4ae2-a590-35ae5358d134", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866468287, + "totalRecordCount" : 2002, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 new file mode 100644 index 000000000..65ab45800 --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-4 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "7e9696ad-f881-479a-9422-5b61f34d0688", + "id" : 4, + "schemaId" : 0, + "baseManifestList" : "manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-0", + "baseManifestListSize" : 1186, + "deltaManifestList" : "manifest-list-5c3ee90f-8062-405c-a97b-b5703fcc06c6-1", + "deltaManifestListSize" : 1117, + "indexManifest" : "index-manifest-c09d3b21-6083-4881-a6f6-b0ecc37dac22-0", + "commitUser" : "c8e2d8ed-8282-4403-a00b-3c3908227c44", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468364, + "totalRecordCount" : 2003, + "deltaRecordCount" : 1, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 new file mode 100644 index 000000000..a91d0a29f --- /dev/null +++ b/test/test_data/parquet/pk_btree_e2e.db/pk_btree_e2e/snapshot/snapshot-5 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "e5a70fc9-5b16-4cbc-9707-b3ed5b83484a", + "id" : 5, + "schemaId" : 0, + "baseManifestList" : "manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-0", + "baseManifestListSize" : 1219, + "deltaManifestList" : "manifest-list-fba7550c-51f9-44c1-af46-e55fb9b308b1-1", + "deltaManifestListSize" : 1117, + "indexManifest" : "index-manifest-d52df85e-6f91-497a-8edd-df92bcc3e772-0", + "commitUser" : "69965629-0d92-4ff4-a2b3-673f8ebb74f6", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468584, + "totalRecordCount" : 2001, + "deltaRecordCount" : -2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README new file mode 100644 index 000000000..7318572fb --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/README @@ -0,0 +1,21 @@ +pt:int id:int score:int tag:string +primary key: pt,id +partition key: pt +bucket count: 2 (both buckets populated: [0, 1]) + +Generated by Apache Paimon Java release-2.0.0. +file format: parquet +deletion-vectors.enabled: true +deletion-vectors.merge-on-read: false +write-only: true (explicit fixture commits produce snapshots 2, 4, 5) +primary-key btree index: score + +Snapshot semantics: +snapshot-1 APPEND: pt=1 and pt=2, ids 1..100 in each; score=(pt*100+id)%10; even tag=keep, odd tag=drop; 200 rows. +snapshot-2 COMPACT: full compaction of every populated partition/bucket builds source-backed score BTree payloads; 200 rows. +tag fallback-base and branch fallback are created from snapshot-2; the branch remains at its 200 indexed rows; it is used to validate fallback-branch TableRead routing. +snapshot-3 APPEND: add (id=101,score=0,tag=keep) to both partitions; indexed compacted files and visible unindexed APPEND-source files coexist; 202 rows. +snapshot-4 COMPACT: delete (pt=1,id=10), update (pt=2,id=10) to score=88/tag=updated, then lookup compact in one fixture commit; deletion vectors are present; 201 rows. +snapshot-5 COMPACT: full compaction rebuilds all partition/bucket BTree groups; 201 rows. + +Fixture construction note: with deletion vectors enabled and merge-on-read disabled, Java and C++ batch scans skip literal level-0 files. Snapshot 3 therefore promotes Java-prepared files to the maximum manifest level while retaining FileSource.APPEND. The source-backed PK BTree policy indexes only COMPACT files above level 0, making these files visible but deliberately unindexed. Snapshot 4 publishes the Java-prepared compact outputs and index changes while omitting their transient, never-active append inputs. These are test-fixture constructions, not ordinary end-user write workflows. diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 new file mode 100644 index 000000000..ddc738c05 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/schema/schema-0 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "pt", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 2, + "name" : "score", + "type" : "INT" + }, { + "id" : 3, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 3, + "partitionKeys" : [ "pt" ], + "primaryKeys" : [ "pt", "id" ], + "options" : { + "bucket" : "2", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "parquet", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866468700 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 new file mode 100644 index 000000000..c4ef63281 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "1f5e235e-3e28-4ab4-812f-6a1e4ca39056", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "b2b00e43-1a8e-4cb1-84da-0f003902e88b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468867, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base new file mode 100644 index 000000000..c4ef63281 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/branch/branch-fallback/tag/tag-fallback-base @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "1f5e235e-3e28-4ab4-812f-6a1e4ca39056", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "b2b00e43-1a8e-4cb1-84da-0f003902e88b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468867, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-3b03f22c-33cf-4746-a2b5-d4652476121b-0 new file mode 100644 index 0000000000000000000000000000000000000000..26cce0f82e71f29bf97d77644006c18554bf2653 GIT binary patch literal 31 hcmZQ%U|TL`MhZ2j6l9;h*JOnqFe{1 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-0 new file mode 100644 index 0000000000000000000000000000000000000000..a221074bd3db27a936c7ef1d9179dbd9b8cddefd GIT binary patch literal 235 zcmZQ!00I_fHgO&nMlef)ftQ5|$YN(>WLH<>6J}usa#&d;`K9GpSinpTEhc3aRv?pu zU7J}zL`q(Tg$>AKV^t93P-S6Z2eMh1Ib}pyIKZM}teQexESx|#D~lqxnkPvf#KObv3xKCjaUNZ^P!1@ K{N)+q6aWA^whizA literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-1 new file mode 100644 index 0000000000000000000000000000000000000000..aab0e6e316bc56ea9bfe338e4e36842224d9135c GIT binary patch literal 247 zcmZQ!00Itnb1{7lWiDA3MlerKgVs#)k0b)lW_6Fi`AWjA1LLlY>VnzlP1s`>=c4mgvj38qe7!sQ& W=7AY##0DrIWH>Sa`Oh=NDF6VW4-RSo literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-2 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-2 new file mode 100644 index 0000000000000000000000000000000000000000..a221074bd3db27a936c7ef1d9179dbd9b8cddefd GIT binary patch literal 235 zcmZQ!00I_fHgO&nMlef)ftQ5|$YN(>WLH<>6J}usa#&d;`K9GpSinpTEhc3aRv?pu zU7J}zL`q(Tg$>AKV^t93P-S6Z2eMh1Ib}pyIKZM}teQexESx|#D~lqxnkPvf#KObv3xKCjaUNZ^P!1@ K{N)+q6aWA^whizA literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-3 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-7f29a66a-2114-4c06-8a28-b10f3cdce826-3 new file mode 100644 index 0000000000000000000000000000000000000000..aab0e6e316bc56ea9bfe338e4e36842224d9135c GIT binary patch literal 247 zcmZQ!00Itnb1{7lWiDA3MlerKgVs#)k0b)lW_6Fi`AWjA1LLlY>VnzlP1s`>=c4mgvj38qe7!sQ& W=7AY##0DrIWH>Sa`Oh=NDF6VW4-RSo literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-0 new file mode 100644 index 0000000000000000000000000000000000000000..01345b0173f5f07aa144bddd8de4dab9abac5e26 GIT binary patch literal 247 zcmZ9HISK+n5JkJEf6uDpHtzeb&Y9XSf27&LnAW&todw&Dde)pd%hwTdQv*i5#b4oYz0(4jh2 z53ZSNzg%0wqiYPi!3-y=CDRq*lZXsSLe^wU4&+QOBGxI1q4Kh${+sQ>@~ literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-1 new file mode 100644 index 0000000000000000000000000000000000000000..aab0e6e316bc56ea9bfe338e4e36842224d9135c GIT binary patch literal 247 zcmZQ!00Itnb1{7lWiDA3MlerKgVs#)k0b)lW_6Fi`AWjA1LLlY>VnzlP1s`>=c4mgvj38qe7!sQ& W=7AY##0DrIWH>Sa`Oh=NDF6VW4-RSo literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-2 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/index/index-e5696b70-4034-43bd-a42c-43e87e0a72b7-2 new file mode 100644 index 0000000000000000000000000000000000000000..c97470bd42b09b11737d1fd6953d36b54bfa8458 GIT binary patch literal 235 zcmZ9HIS#@w5JheLZ6^W37Q()See3CwDCj_<;{cR!Aw&V^qU0R>SX3G5jou%PEe!+< zYMqZEory*PIhr+9IWtKSib12-4x50}+-Saprh2xfR;_WV55r`%#iTrdr9(v>51yO) zdQqvvr)zYl!H^;X5|M;#$d>HMiCoB)q&reP?l0vYEAf=+CqjHb@y$u5z5hAP%+tFL H=iTuK|4Vs#)k0b)lW_6Fi`AWjA1LLlY>VnzlP1s`>=c4mgvj38qe7!sQ& W=7AY##0DrIWH>Sa`Oh=NDF6VW4-RSo literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-11be7772-670d-4dc1-adbb-fd8413f2779d-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/index-manifest-11be7772-670d-4dc1-adbb-fd8413f2779d-0 new file mode 100644 index 0000000000000000000000000000000000000000..2f1aefc19e1aaad811469f7d84faa16f7d6df0ce GIT binary patch literal 1647 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~m$;^Zc2#cUG6BZZEiQ*I9;jgH$gr`H5a4NVo;~|Nmp!9TW=?VCp_03Q%sqab zt#`IG|G)oFypF&*@zdw|9ReMHFRAMJbucbxdD5N`y=^&)+g^Bz$8ujd`g7Hk?z3?( zgYK*lx_u+)_S9L|j=hy(X6bh=beK8Uq>)jm&tSpY{&Ocg6IT~X8t)YJ+?K5tcY^Uv zsM|zS3-vpyviad@(UYfWi}$V@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~meR{q%zy0bc&5lR zo|4`w%6;UpX6xE_uWm0}wQ23G9KMw{Jl409-A)S#Hk&_cVoDZgS37@knc?{zS9z~% zZkripK3#9>sVJ}7_Xd~MCO)h9YS(t_mCve}Ri0YH#b@Fe1sPcGanC6)FuKsm_@1@w gpjUH+LNbf8pvBXIxN{|Jje?94>`$gOTA+sv01@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j z%u7cx#?8~mHQvwBmk3qCo>54ua`Mqk3i1z+clHnULvj;_8W&d|SCGeofkEUP;vWxdL7&!|dRJtPP1&kyENyl!Uu0AgDo_I11Fl>Sx<6-KEPf<$XD8(0M5EuBmhB(5# z1CK{YzQm_H*grJL*)^Ud?O0MlY^?~mBsI}&N^F?{N8Q5JBK3Ysc=Kq zkF%P${nfebe){+Ekn=s2v%KREZsp|U;ow(L646mnvS>&U*qE^)!hwO`M&KjEq_y2I z&lDfoxo_I-?N3WW&CWYvzOsWNFP^R~@-xe%1H47GH9fuAcqg z&0I8rfuEa&-^GPZ$x1}wMT3XH$wI~a5@YSFmHk_zu3SI<_i*}ojhB^cn?5a-^W6Hm zN^$bJ-Jez!zlw`4u5XXFt`ipFmE=-JM9)6cWsFJ;EcdwQlouFX z=wy7)T6WN@xk4d1;P$Q-Nv7wrb9hP`dCH>M9BLR0C$u)Mm?QgbOGECp$cH>S4l+9e c!c!)%;Sp+Jkx*)wH{bAY!yO|5cJ#Oc0P0Fof&c&j literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8727f0fe-6af1-4c2f-8560-c38dd0a81ec0-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-8727f0fe-6af1-4c2f-8560-c38dd0a81ec0-0 new file mode 100644 index 0000000000000000000000000000000000000000..c09730c364b989b42f2aa37956cd53a1e96782b5 GIT binary patch literal 2149 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8R#o2-m*CXDr#7+c zrt4Y3#VlW#G`8yhW$@@_WhgqSptivD<^TUXwO3F6YbTv1k-9lcIy!Gs_NK9I$q>xaQSrdW5wk^KWprm zPyXO9m7etf?EIN?n_RDSGCy);S<01{?H9>=X!%7;sS*e7R)i|o&f++ C#LHa( literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-bd400d22-3691-47bb-8262-753eb1a8c2c4-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-bd400d22-3691-47bb-8262-753eb1a8c2c4-0 new file mode 100644 index 0000000000000000000000000000000000000000..d055bf2477d99a61eee17c688bbd616bcda0ffe3 GIT binary patch literal 2254 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8?%0M;N|3C#{ISq= zO2vugPdJXTXl&L0%iwX9gQ2P4P-}wb%lGfzty?K+y7x%Mk*li~YAfk%SDCb?czI#| zrju&3SzI6eX>i`6=E)_LE&P7+>K&OOXI;}~Zg|=L@Rsi}$4sGx8m~*T&3ug3EX>-r zL&5h(pW3=>y(>#n9~Jx;o^g;T&4Eo#DvrZ8^JR$Z+qA^>elyO9m>roFGJkh78wYn+ z_wW1<|LgY4AK2O^@S)-de_%(!DUH=NDs$Hb~~klJ9KXcY)xHg zH0MI1>kfl864Sc0Bi|i+f2zL8oJm`up!twm?k0vt26=&of?w?M2NIOS434UJCOp{x z*yxN>q>rTj*$mJ7#ufD`=}M*oA9)|9-F)Np+fsF*J)@|ogJin9@e)>fftyEPG)kuT IaifPC08>T)0ssI2 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-dd531b47-e547-4b27-b799-09842a4b026d-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-dd531b47-e547-4b27-b799-09842a4b026d-0 new file mode 100644 index 0000000000000000000000000000000000000000..97490cde9e59f1190d4196b440d22315b642eae1 GIT binary patch literal 2106 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8HoI!Cke;$%CjOD& zqA;`R{!H_jG`8yhW$-X%VQ6xfQeL3`^56gZ-=g)sCoak9UdwqcY<@Y+G{-peY+%{X zuUq+FZd&^P?e`5C9!itfS;#qG_>_vz*X#d+0Es JkZlLL3jtdKx<~*3 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e6ecca13-a405-472b-b1a4-77de887ae406-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-e6ecca13-a405-472b-b1a4-77de887ae406-0 new file mode 100644 index 0000000000000000000000000000000000000000..db786b0451d6e6b3ebc04cf9c55cefc7c54494d2 GIT binary patch literal 2591 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8Y;44)*-f}p+GdhD zDdccnl*9{8jjj5B89ZWz8N!xin-pYUVw73ic5h?Gb?rS0pV#=G6DX)i45@!Q4B9H{?1vo(I!9B?7K~6e{q%>U$Q`N zBP*Yok&Q%%f{OzO%f$tb&1?;s58^NCZLF{m_}G}y!>Q2VbTnFHL564Jo`9?N*W7t+m@{BtHCj;Q#-6w&SL%VH!Kj>$2zU@wj|7-zxS_l$(Tv$^(`-g+)*P zJGeYy5>(~XkeS*c6euSoDiF18JyWLbwBK=?uf|*1X6fc!4488D_TIHO*QsT{P1%#7 zRIVRc9DhqzD{WI^!nUCXV(3j`j&W6&r3`eZ5U= zS=_SFbGDwwJL~O4C-d)G&%Zk9+(w_=(tp`!+oMiToAZ5Q{+yiltJUiw!_%v7uMEq4 zm!!GfbjvyGV?GuOI*+lqxhPHK?BvYZc>8)((D$2SkDjn*ZgSERuBu;km9?&9TI{#d zn>xKH)VFi)7Q;>j+QU|^A>#VEtvJwbX`%!&sLe-3mm>H4@_;7fX9PjZEF0q>d@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8y6@*Y1Y9&b=P3Fn zt7wb=Qh{G=8e8@MGI%WIWni1-t206KW&P%?n%Y;Uvgan6{af!5ZEEoMxaTt6#H#<= zUNe_W33}G|QtLghkH;iUR|aPWdxr9giV6;fXlH?jj}P8Samm#bmh+W&oKWC-q;P=A zC3(Yxh0?4hzXPs@X`L=PeWmA=Yu%)u(@ahXNA0>P@OHn2O;I^Zf#A9V*W?2%Wud1E z&%DlixA|0feNgSvoTK00$aO?KWRmNga3l37@62uYCn+uqvY(rByjm|Tewn!52M1;j zDUF;1yNwGNSI(%;v_Ij(;xy~;dZVB%QOn}zE_-t_ef7#)+ozrr@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhE3l(cGZ!}E^mDy(J`WzMlSFv*E(cNeU+y>e0Fo9@(oCMoBnwoOj&e3Br+ z#9fi=ZZ6n%W6|o=OQNE-r<>=jY+bSS?6;rqYPP+9=QCr2ZGy7G!PQ<{SE!UpoV)zQ zZs}tE4$FBv1xlByD{KfWXyD=K;$(1OV4Bm)*Dm}pqo{*nN7H8pgAZv8T@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhRH>Ti*9e*dF;dl?>PzDr%ITPFlubo|I6TUmWe^gRaowTs_g&&`QO6dGcjcA>~x*? z>y_H19M5m8-)xm_-->K2e1B~3?|pCo@9TYgNS%vavgzD2r@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhOX)-I|Ob9d=5CYGa~KCx(Q5s7&W%)|7Gx4%f#^Lh@jMht(O1!FBl5>-`?xAO3GZ& zRPp78f>MveTY|T}|G)3;|Lq$$##Sh>sb9T(UFOydo@u+bJk<;C?3)@SI&blVWeNhq XtPjLoI9M1Igy*m~G@3I9pj!w4i8OhN literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1 new file mode 100644 index 0000000000000000000000000000000000000000..ec68268f533b1a7166c390aa3cfc60bc93984179 GIT binary patch literal 1123 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhFNwmxpzB7uYP?ag0<_TktEX*Mvblde;GW^GBGe66_VXxD$B#nIctfG9ACxhyBsqQ zf6ptLzVJUYf6ygA#&4V)@BZ(5`~UmLjlK#dJX~j%7pZRB<-KvI^1%@0!pTWH&Tf2m a&DmkIe`1Ma3k!pS@ErDrMswx>bV~uj>3019 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-0 new file mode 100644 index 0000000000000000000000000000000000000000..1212f12e54963eaa29f889637d0c7bc8771d2948 GIT binary patch literal 1158 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhMFYC2W>xZa;Wn0&n{7z_l2d4Nn@-2Uj~n-%nX;g5_fLPc|XrxyuQvOC&^)hGot_-gVG!)K0$`?0yze@ Ldpr(|=@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhR%-`kQPe5Z_@ryb z?JDDWjV#hCGnu|^{FeN7U;gj^wRX1|`41==URLS~(iBzkaSpgMiOoYyGG=!5slX7G ZYf=I)0wfq&7!-u(us1ZCGY6ns3IL&0aMl0- literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-0 new file mode 100644 index 0000000000000000000000000000000000000000..fde5da56a71cc8ed8e94ae284a08bcc32a48a231 GIT binary patch literal 1006 zcmbVL&rX9d9M;66@nDP((DT9r>|*Muk*t6jFpUdoI_NeGf0ojj;KD0-@(p|sPd)@?H@kbs^p+<|>P=WgUk~O$WK#y(fHN=QK4^iMMvZkk zrJnqm@UcRR?+H_;G>DQ|S%{5U%s3TH(nkRc{S>qZ^KT{6M?Fh3jbd}@6~qGMtqO3a zVMGk5qY9D=9>n7&%F?cDs4!W<7>-V$sv?z0O{>ydi&POt_6FDs*t0deh(IhG+Lh`b z6ngDJLLexvoHij-^=@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhLvR%rZx(eg}eVwIQlosPl{;|qsCVKzYHF0nHU}&5tLf6)$%|81w$eK+k2f>Ntp|p zD!$xMQ0j4bOYpY$|M$K9zkTDz*a`(U^{bb!%iNm5Gi}$Fr+UGieN%%(=PiD)OhG`H W^?{fR2MdFO@ErDrMswx>bPEBk0d_V3 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-0 new file mode 100644 index 0000000000000000000000000000000000000000..de2e7a181080739ee83f216e95c57aa3c9f34040 GIT binary patch literal 1234 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhAR%9&MiL!f`JkTb_+>;is4AYgI3+Id{b> zt8}-ujf>45YnLP1>vrsJG-jIgM=P|~D~bJ3_~O4)D&qSr*X=SYU#hOKA*`T*hl7ui mhrvOUspw{+lt2?h`ohBxT`XqFC^4*iV8g((N3wwl-H!ky8>;jG literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/manifest/manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-1 new file mode 100644 index 0000000000000000000000000000000000000000..c2665104d6e428a8508fcbb33f1b9e5bf294c6a0 GIT binary patch literal 1126 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zh9_xWJV*8K99|z%+WO=B+?AO3%T z#_4r`$_{?7wmact#JWwjFZ$At_qYGm&punm$#CkhjM0u+nrjt52lQL1de}5M?mc*M etJqsZv1$hmlRyV176t|3IqVIM=F9=;_5uKjk$zeL literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-2826504c-fd4a-4ef7-8f63-724e727201e9-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-2826504c-fd4a-4ef7-8f63-724e727201e9-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c2b4891d029c38c40ed45639d46a7cd9ec0191a7 GIT binary patch literal 1409 zcma)+Pe>GT6vv-44Zs0_1_1Q8uoS!Z_3r&cb+WSn?I@$ilM--iYvZL< zC)He4M7I6z!K>!|WJrq0{-0hOYiHkg-aI|*!D1{p#TbBqFp-EpQf1xj)92N-dU2&D zmQZ}daVoe=X%&=&M3wL-CMeu@%e9j$iNK7!CJODMDe9(N&`nd+y?=~xvA+GyDwl$yt_ugQ>Q7=li3NG}$T4h_*HV`^$fxvW z@xbSNC?oGfjn|}jJh~JvZI~DJO`_5N!5itM;VN=cyxrJ!nW_8-@%VF*3%bnLr8fNZ z4Oa@)TZ_)ZshO&F+o{f%=Vmpd!^m``wN!Dje6yr=r#tPgUL$RHWjjlqXENz7$LKY( gS)*V(#c9X3vqsUG&KM_^CA_NjKe#FajN!-k52@_VzW@LL literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..81cdd6d60483090df01b299b1d0000d3cf382b8e GIT binary patch literal 1878 zcmc&#ZERCj7(VCrw!33(SA=uyHCIx9^kz&PyB+&t#cYh~R&Wfq-e_V{yY2#2M{76y zwI7WU)Pz724K6bDhlw8<;WJ<|(7@U(-`dn?@!{&~`K z&pGcs=Xu}fdEax}-L_RBh|F+g9Fv91->j903TuG~2vwb0{SWHc%b|(jGPmdJgLunG zXphCw(*Io!_n;tP67pBLq&4S;+xRz8$~usZiqeE=Ci(9d^&Vaod+@`10A7 z?^b3zr#q&ey!6Dyz~qKZ`-SxrZRcCh9sT5sAD%xsGj=L?DER(YAD*0kwz{mkmqQlJ zeDFZTL+0kLe9Zd^NBBG^56U$?44z12{FOhBm$M+)i2`*O>{3mfKR2(2ll88dO38~EYNkKTFKMo=45sZ z>q6N~px& zqFZ>>m=?WeZrLd|6q^k&HY{Sk$44H7DObFk@d9i{%P>n+AhQ@^L!w0IF`3HGQ#=V8 z=Crjiy>2ykUlqun{O;3BNFO6eXE^3?Z-*lhi4DYeh6kfNw1Eh5g4K)MW#Ns1XTq`m zPzss=w*qXga8F=MDA3g&2zP~^S|8XNx|_{rEA;~vVYO{jC=lM**|ovDqW*98MmoGq zSehdSr@6#Xh;d|7z-8XOoZC7WjVHAbkBA*smZ8XYFRHgm5}cx{D2c-|k|0d407~a$ zB6h0M*DZ7w%Yt9enw~<6s&YBhSD+JoFu)D+1K6yCP&!gUY7j%@b;J@zIq09QbPCJ! z*l(uYU8n`{Q78t}`d?ouCvnq@73A@pFNmHOM0c@3IXBPIG?r7mJZ_QnBGR9X1h7Z? z3W=*&)dLA&!Uo1=#0gwpwHsUve9+f~OGuqKpTiKqF6!8`IlQv3r2%9Vb~q&oV&zfs3lLK+_e=;maRPM3OJ<(uN)z zNP1q@l7q4MPLEQfG}P33s`_@tUhMa*sr5zcnv~jTU8Aqx*W6HBrzuTJW1|v@YJJe3(D4IjF0_(%O0^V@Ds literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-1.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-5a5fde13-5159-4129-916b-ec43bbf7c968-1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c2b4891d029c38c40ed45639d46a7cd9ec0191a7 GIT binary patch literal 1409 zcma)+Pe>GT6vv-44Zs0_1_1Q8uoS!Z_3r&cb+WSn?I@$ilM--iYvZL< zC)He4M7I6z!K>!|WJrq0{-0hOYiHkg-aI|*!D1{p#TbBqFp-EpQf1xj)92N-dU2&D zmQZ}daVoe=X%&=&M3wL-CMeu@%e9j$iNK7!CJODMDe9(N&`nd+y?=~xvA+GyDwl$yt_ugQ>Q7=li3NG}$T4h_*HV`^$fxvW z@xbSNC?oGfjn|}jJh~JvZI~DJO`_5N!5itM;VN=cyxrJ!nW_8-@%VF*3%bnLr8fNZ z4Oa@)TZ_)ZshO&F+o{f%=Vmpd!^m``wN!Dje6yr=r#tPgUL$RHWjjlqXENz7$LKY( gS)*V(#c9X3vqsUG&KM_^CA_NjKe#FajN!-k52@_VzW@LL literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-6444b55f-03c4-4cdf-829a-1cd0c9d670ce-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-6444b55f-03c4-4cdf-829a-1cd0c9d670ce-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4fc0fb7f8a7b1c00f6dafc3b873d3bb1047e00fa GIT binary patch literal 1411 zcma)+O=uHQ5XYyx+jOb5AK3TE!XkS~h+t#3A4!E$Y&B3@4YjfX>tRVYTMgParmd$Q zdhwu?f|r6G#bYia#h!XD(vzT9(VL*qla$a>XLdKc8`DFV@ZOu*dB6F;HlkrbkxLtj6;%6=X1zUg$6Mwow$SpWfPA|XAd%7)p&x7to?aHSC} zVek{%uHY`EPf(%~U4=I>LE)ZTZX8{S`peL3iH?iAXbQVY3%Y3vyU)%+QYf0Ha5Tw7 zH%*>f;TV#pNY)#d>Tn z7FaiWV-W2yXR;5>a>>}h2O}l)^_usy_l#`rX4cFZcPF!R#_{RtX^*?%f|;MVV`R;l z+qqk@%OiDqqt-6Tea?|^w@_I%=ybG1u0Ts6+P-2fS8YQA)`gb1T6h$Lk%+3|E-oml zvK|L4w-S@_dK#Ko7Z-S%=i>61TQqm%kZ?-qT6jU&)0hOt5cR%;_46+hO9JsEm)CM0 z2RzR6@%&g&>tKZm{0<1b&53p?(MCHPcM^HR!*}D0ueG6i>Kb<>^?}t4J?LPZ@3njC zJt*F%&#DuPTFo7a6gi-kOBC12PfOZxYS0=Q)l=3`X0SAPJ)Ih| k^-(>O(F>McT(B)GqZjRkw0=cd$5w6q!9%R&EPj6f0cy+32mk;8 literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-c469efe8-01bf-4123-a9f6-e9505701ffba-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-c469efe8-01bf-4123-a9f6-e9505701ffba-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3ce73a2745d2007e663afc9fe2609814c047cd95 GIT binary patch literal 1885 zcmc&#ZA@EL7(VCrw!J`GO4vE+O>SoQqc<#5iU_oyLzrCa3+1? zf!}|=G9NuPeB%D!u3nn+?@uqZR3nTKcuAPa3?>H`PXA#mam#aPb;)jXzH(AT2)%gg z*bJ8mCOa;)pMT`*htCC;b}ahOZa>p@x;63nm*0H<%=~ws91p%5oV)nZ{DmIJwmlp& zVdjJfDsFP^+VaP|k8p&~vvLC{G(8TUNM!EF)wAU+3U;DE-U)lc2F}fsuO6$s9S4^0 z9Or-1nkQXkfE@Ya*AFc|vdNI46|>M02OXqAX6P2^z(3op@GOIFg04%|3JP6$^Op}C zx>OcJD%ika!kD)^y?f}`A3_Ph0_Js~W z6X0fm&8hAV>&s8FA1do&bKcXo8{@N91QldVXfRtr^G zVsM&^jfEIT_6oSnyN7d`N24QgEyN>YgPCb8^0WsvSR@Hf(Xc3q`(z|Rm>vO?&c{S- zG)yNv7!r)N!1Aw@OW9O^UB3BDTO2KfN&rkzlFw1U(khRExPC5*DrYi63n@;dez zX?GQB0elpS!La_<*BcJvqL(YkgIQk?Juir^Vu7-5o}!1bJkHArlcX1s{$wP8J9UiCWr4m|u?fEZ!B`~z{9bMBo}sw= z1uZ^0I5O;3YL&*?I(JR)-ofYk-1pUaqxH>7U9`T*+vja*tgF|QW~He~iA1&Dek~ep ZQhK%iMrEtG4~n| literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-cda2ddb8-fcd1-4298-a4a8-4778ab7aa7a6-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-0/data-cda2ddb8-fcd1-4298-a4a8-4778ab7aa7a6-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3ce73a2745d2007e663afc9fe2609814c047cd95 GIT binary patch literal 1885 zcmc&#ZA@EL7(VCrw!J`GO4vE+O>SoQqc<#5iU_oyLzrCa3+1? zf!}|=G9NuPeB%D!u3nn+?@uqZR3nTKcuAPa3?>H`PXA#mam#aPb;)jXzH(AT2)%gg z*bJ8mCOa;)pMT`*htCC;b}ahOZa>p@x;63nm*0H<%=~ws91p%5oV)nZ{DmIJwmlp& zVdjJfDsFP^+VaP|k8p&~vvLC{G(8TUNM!EF)wAU+3U;DE-U)lc2F}fsuO6$s9S4^0 z9Or-1nkQXkfE@Ya*AFc|vdNI46|>M02OXqAX6P2^z(3op@GOIFg04%|3JP6$^Op}C zx>OcJD%ika!kD)^y?f}`A3_Ph0_Js~W z6X0fm&8hAV>&s8FA1do&bKcXo8{@N91QldVXfRtr^G zVsM&^jfEIT_6oSnyN7d`N24QgEyN>YgPCb8^0WsvSR@Hf(Xc3q`(z|Rm>vO?&c{S- zG)yNv7!r)N!1Aw@OW9O^UB3BDTO2KfN&rkzlFw1U(khRExPC5*DrYi63n@;dez zX?GQB0elpS!La_<*BcJvqL(YkgIQk?Juir^Vu7-5o}!1bJkHArlcX1s{$wP8J9UiCWr4m|u?fEZ!B`~z{9bMBo}sw= z1uZ^0I5O;3YL&*?I(JR)-ofYk-1pUaqxH>7U9`T*+vja*tgF|QW~He~iA1&Dek~ep ZQhK%iMrEtG4~n| literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-0f061bb7-3867-4575-a7f3-a649acff590b-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-0f061bb7-3867-4575-a7f3-a649acff590b-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0099628a400b69f402cd44a15ed9669b6ae8c2f9 GIT binary patch literal 1932 zcmd5-T}&KR6uvV%ECajjE-Cl!PBNiOA|oV$x+tJFCk>AHCT- z_ue!2e&>AOId`X_y2*tRo;2WP2EX^g!ucE#%V6L!k5JK<5B!b%c?QG|+U=`Hk1%M0 zOtdsVVdr~Z6P9N0sTuer@8HVKrQN|jHy0bXuH3j-vw6>rh3fe7_4zHA+pf>KCYIOF z6~EiIK2vaTdF`|iY+IYo-MYLwm9@ETb&`qy`SnA8{qf`RFoPI`z-xadKE~i*e?Iwg zc3Kz_7KP5(Otwp%Mss=Vg@Wg^=D_Pb7u0!7f?Hxi$Xk7uNscz-x`$UEusr^1$H+ydVEIb7~CeaN@vuib27?Snc=LqvMOJJyXs!Q zz1Z#Viyg{~8HU^q_Rt}DD5Q$(#Fcntc=~ON7xYgisUIocBi&sxjpFHORLKRx`Phxm z{<0P~rFnfp4Nztn>uBWhuihB?7#s+>tl;1%F&H8+U{)ZSnNm@j;NU&lH03q*OHa`u z)l8EPW;&#rH};VLp^~QVro;e*^3u#A;qf2~u^~Yu!z{kSX!CIn*J{)GjO6sNQQv(c z3lAoCkD73V5+sub>hN`cj@%yW=;{pig!e}~+OY+!g~*W^tn=*-#v*}!XgpjNu-Ssm zzQ%yBe!DMNAK10c*A%!DkH@X*0aLKGdS}2FtgETtQJ7zGo4yf0-NffA#Nae5_Xa3O zIt5&oPFY7rPq?c)8sHE%!^+Ux{$wGlFp46aBC;R~2P7myn1wt@nurOTiA>fFWRR73 zubMSAg#;BTIaF7m6a1-w8{`GBB?~5Gz>G@}MdVb(G)4;aSSGo`NW-jGPrD;o3*aMJ z47&B-zOGuZgN&K+!-_A6p5p~asz8dHz2r?s8t0^222m{{^+`zpd!(+Aa33vuAOTG1 zK)DQ9fy-)wi-r&S>ag<}(kmGNON&r_QdoVpA?!O8_GpeKktRCiDn;+Z#=92$m=#2r z)6pt8-Y97KRuHc;WQ3JYWJ&My5GlO?x&ZoW!$@Z+_U#0=Sx~WQ^(e@1F=UF7MorRG zw&(`l^cF##z?DizqxS%jpR72ad}smcWN{ObC7#?d<-rG`R&MWpwlCUye@D0TxoCG! ztgF-MDsj0>N}WZazSuJn=YyqX;qppXX}H`|7Af1}E-jC`DqS9rt34bI?Td!P9#<&3 V&+RG}4nR?-|Ika|zX86({{Zm~n+5;? literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-42b50aa8-9cf7-409d-a011-f7d6a0cad4a8-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-42b50aa8-9cf7-409d-a011-f7d6a0cad4a8-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0099628a400b69f402cd44a15ed9669b6ae8c2f9 GIT binary patch literal 1932 zcmd5-T}&KR6uvV%ECajjE-Cl!PBNiOA|oV$x+tJFCk>AHCT- z_ue!2e&>AOId`X_y2*tRo;2WP2EX^g!ucE#%V6L!k5JK<5B!b%c?QG|+U=`Hk1%M0 zOtdsVVdr~Z6P9N0sTuer@8HVKrQN|jHy0bXuH3j-vw6>rh3fe7_4zHA+pf>KCYIOF z6~EiIK2vaTdF`|iY+IYo-MYLwm9@ETb&`qy`SnA8{qf`RFoPI`z-xadKE~i*e?Iwg zc3Kz_7KP5(Otwp%Mss=Vg@Wg^=D_Pb7u0!7f?Hxi$Xk7uNscz-x`$UEusr^1$H+ydVEIb7~CeaN@vuib27?Snc=LqvMOJJyXs!Q zz1Z#Viyg{~8HU^q_Rt}DD5Q$(#Fcntc=~ON7xYgisUIocBi&sxjpFHORLKRx`Phxm z{<0P~rFnfp4Nztn>uBWhuihB?7#s+>tl;1%F&H8+U{)ZSnNm@j;NU&lH03q*OHa`u z)l8EPW;&#rH};VLp^~QVro;e*^3u#A;qf2~u^~Yu!z{kSX!CIn*J{)GjO6sNQQv(c z3lAoCkD73V5+sub>hN`cj@%yW=;{pig!e}~+OY+!g~*W^tn=*-#v*}!XgpjNu-Ssm zzQ%yBe!DMNAK10c*A%!DkH@X*0aLKGdS}2FtgETtQJ7zGo4yf0-NffA#Nae5_Xa3O zIt5&oPFY7rPq?c)8sHE%!^+Ux{$wGlFp46aBC;R~2P7myn1wt@nurOTiA>fFWRR73 zubMSAg#;BTIaF7m6a1-w8{`GBB?~5Gz>G@}MdVb(G)4;aSSGo`NW-jGPrD;o3*aMJ z47&B-zOGuZgN&K+!-_A6p5p~asz8dHz2r?s8t0^222m{{^+`zpd!(+Aa33vuAOTG1 zK)DQ9fy-)wi-r&S>ag<}(kmGNON&r_QdoVpA?!O8_GpeKktRCiDn;+Z#=92$m=#2r z)6pt8-Y97KRuHc;WQ3JYWJ&My5GlO?x&ZoW!$@Z+_U#0=Sx~WQ^(e@1F=UF7MorRG zw&(`l^cF##z?DizqxS%jpR72ad}smcWN{ObC7#?d<-rG`R&MWpwlCUye@D0TxoCG! ztgF-MDsj0>N}WZazSuJn=YyqX;qppXX}H`|7Af1}E-jC`DqS9rt34bI?Td!P9#<&3 V&+RG}4nR?-|Ika|zX86({{Zm~n+5;? literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-d1e79d3b-bb1f-4afe-a6b3-8cc85490af74-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=1/bucket-1/data-d1e79d3b-bb1f-4afe-a6b3-8cc85490af74-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0099628a400b69f402cd44a15ed9669b6ae8c2f9 GIT binary patch literal 1932 zcmd5-T}&KR6uvV%ECajjE-Cl!PBNiOA|oV$x+tJFCk>AHCT- z_ue!2e&>AOId`X_y2*tRo;2WP2EX^g!ucE#%V6L!k5JK<5B!b%c?QG|+U=`Hk1%M0 zOtdsVVdr~Z6P9N0sTuer@8HVKrQN|jHy0bXuH3j-vw6>rh3fe7_4zHA+pf>KCYIOF z6~EiIK2vaTdF`|iY+IYo-MYLwm9@ETb&`qy`SnA8{qf`RFoPI`z-xadKE~i*e?Iwg zc3Kz_7KP5(Otwp%Mss=Vg@Wg^=D_Pb7u0!7f?Hxi$Xk7uNscz-x`$UEusr^1$H+ydVEIb7~CeaN@vuib27?Snc=LqvMOJJyXs!Q zz1Z#Viyg{~8HU^q_Rt}DD5Q$(#Fcntc=~ON7xYgisUIocBi&sxjpFHORLKRx`Phxm z{<0P~rFnfp4Nztn>uBWhuihB?7#s+>tl;1%F&H8+U{)ZSnNm@j;NU&lH03q*OHa`u z)l8EPW;&#rH};VLp^~QVro;e*^3u#A;qf2~u^~Yu!z{kSX!CIn*J{)GjO6sNQQv(c z3lAoCkD73V5+sub>hN`cj@%yW=;{pig!e}~+OY+!g~*W^tn=*-#v*}!XgpjNu-Ssm zzQ%yBe!DMNAK10c*A%!DkH@X*0aLKGdS}2FtgETtQJ7zGo4yf0-NffA#Nae5_Xa3O zIt5&oPFY7rPq?c)8sHE%!^+Ux{$wGlFp46aBC;R~2P7myn1wt@nurOTiA>fFWRR73 zubMSAg#;BTIaF7m6a1-w8{`GBB?~5Gz>G@}MdVb(G)4;aSSGo`NW-jGPrD;o3*aMJ z47&B-zOGuZgN&K+!-_A6p5p~asz8dHz2r?s8t0^222m{{^+`zpd!(+Aa33vuAOTG1 zK)DQ9fy-)wi-r&S>ag<}(kmGNON&r_QdoVpA?!O8_GpeKktRCiDn;+Z#=92$m=#2r z)6pt8-Y97KRuHc;WQ3JYWJ&My5GlO?x&ZoW!$@Z+_U#0=Sx~WQ^(e@1F=UF7MorRG zw&(`l^cF##z?DizqxS%jpR72ad}smcWN{ObC7#?d<-rG`R&MWpwlCUye@D0TxoCG! ztgF-MDsj0>N}WZazSuJn=YyqX;qppXX}H`|7Af1}E-jC`DqS9rt34bI?Td!P9#<&3 V&+RG}4nR?-|Ika|zX86({{Zm~n+5;? literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-043439a0-6e5d-44a8-a6b1-e43e09d661b5-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-043439a0-6e5d-44a8-a6b1-e43e09d661b5-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..972c59da9a8b83c74476816527695da82846276c GIT binary patch literal 1432 zcma)+&ubG=5XUFGn`G76AK3TE!Xk^15W&`LYm-_^vDHAe8e(OG_Ohg#Z5yy{OjA!i zcoHeS^wNW;9u+DcME?L0X+0DV3Z4}4Tw26asMML=%_cEDbP4axo1OQW@4PqJ?C7M< zfXPf_YkP0jJ5(k(OBo>k_VQmC*8olfKmY*!oh;6(KUROXw}f2-=)o|0+)yI!@9(~p zT0%7^sv+$Cmz~$Ajtdi-XeZFu%`UN>UwhAGmlk4v8Cd|LiGlk*^8y=UA76c454Hq9 zs|ANKc)3)_SL^~-rF02OM55F1CMGD{bIY~rm56UPyq0hi?V_pCO`6b6Q=|L%91>~A zXqp*luLdpATj<;5O|Y>q1A?UZnB{A?*- zoL^ipm#jH^KF`P-g-F<(NIx)(1>*@mGB62!o#wstT_c^jo;EYaow4+!ad>oe)ZuP8 zW9CM08)fDh-+_^}1g$IPqjY(h_QSZX6n}3s7 zFAz_1c`fxg;1QmS<%a7wUbirT-v@PXbD~d_=xLCRIEg&rjc4PF-wL35EEC+3)Yn(d z(C=Z!`G=sV-h<+K`m8FksNvj^SoKvi>kqMh|IaEGeAc#b;!|Cfk+m7}bq&ZNye9Wb z9o`t9ifxF+*l{t~`{%obk*89N<2KxQgG z46z~LFGMcHWd2#|#t-7HQoj6X*{<}=m$k=sd8xR#py|DOvNxf{rSoQgBzA9(js5lD6FeOr^wi9YC=*=xdx?A#>q}S!yK2KQpo0Q1N^9eS6#9y7EJRdD3&w zJ?}Z^dEVza=iGL-cDX2_YdoFj=o^15Em>t6gn&pzLaNW){TFGQfru~2eH_l-E0PMy8) zmmjXqMa~VKz4zy9S7yD3Q}fN0gd+r4;zl~n(WCR{f3pGkU??+YHvQz56y37}B4JOnD!*^}2UlwuaRA}`(!bAmd~!s_zrlG}0k|AO;B zc`eXR+D}h@{L{N858bBQ(88ew1SNnq#0b>{74&C{8J;Cj4Nx_!TEL;Jul@LrO|#0R zTLlvkCddg^yZZneJ!X2{OGnh>6E8L&-Qjg~xits@wxfy8Of62>Jm7VHLyl+}^u~sx zn90+s8r2wa`uO^{pOqIe!V-;P@l?t5{Hyr9%plAEf(>dpWdkkamaLX6KKQG{P8pTU0>I)S)Sd{d_qK$cN7#4)ESe5`Rp9rU@n0K$ z1x-llX!U|Gy~|W6c_1ku+>PJsIRljxBOZ zyi{2)ksCla-Xe#9SE?8d?*U;il+*1jfq-nq&Vq`_-YVM$Uw?lz9DDYlI(pY&%=w%e z8yOfLa=L0<4K=mS>YjrG&-6O)sdY!{nq0M!x<+@eySbsZPIWc88XH~Vh}zSqMk0-_ Y9<{H*wNp9-hr0Mjj=|@$6aHBL21u=amjD0& literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-17bc0233-1c62-46c7-89f1-6093f22e9eee-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-17bc0233-1c62-46c7-89f1-6093f22e9eee-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..972c59da9a8b83c74476816527695da82846276c GIT binary patch literal 1432 zcma)+&ubG=5XUFGn`G76AK3TE!Xk^15W&`LYm-_^vDHAe8e(OG_Ohg#Z5yy{OjA!i zcoHeS^wNW;9u+DcME?L0X+0DV3Z4}4Tw26asMML=%_cEDbP4axo1OQW@4PqJ?C7M< zfXPf_YkP0jJ5(k(OBo>k_VQmC*8olfKmY*!oh;6(KUROXw}f2-=)o|0+)yI!@9(~p zT0%7^sv+$Cmz~$Ajtdi-XeZFu%`UN>UwhAGmlk4v8Cd|LiGlk*^8y=UA76c454Hq9 zs|ANKc)3)_SL^~-rF02OM55F1CMGD{bIY~rm56UPyq0hi?V_pCO`6b6Q=|L%91>~A zXqp*luLdpATj<;5O|Y>q1A?UZnB{A?*- zoL^ipm#jH^KF`P-g-F<(NIx)(1>*@mGB62!o#wstT_c^jo;EYaow4+!ad>oe)ZuP8 zW9CM08)fDh-+_^}1g$IPqjY(h_QSZX6n}3s7 zFAz_1c`fxg;1QmS<%a7wUbirT-v@PXbD~d_=xLCRIEg&rjc4PF-wL35EEC+3)Yn(d z(C=Z!`G=sV-h<+K`m8FksNvj^SoKvi>kqMh|IaEGeAc#b;!|Cfk+m7}bq&ZNye9Wb z9o`t9ifxF+*l{t~`{%obk*89N<2KxQgG z46z~LFGMcHWd2#|#t-7HQoj6X*{<}=m$k=sd8xR#py|DOvNxf{rKchI=0k1j(hL!ko?iv=hc~)56?lncFgO9zvUDT69}&J5M_Z zq6hIbMDU<^bqu^X5RXHKg6t$7^{Ox!h=(DQ`CgJHtyNqo-}k z@aXAAMg_q=$&AI<&-`K|8ewr%9w)gtUVb5m-f-ELHn251e15}|GK73kRZ!|7ejU_a` z;W$OSrL+r5RH91w8wd*b)p9j+B^rn|uZcppXscCIcBrPUR{eJ$|AJ6>2%*h~s9c6| z42>x&pN1JSXXQQc$50z% z(H3hgeZ?x|%{%y@rG&aRYbt%&OlQugt&DkTI6Z0Z?eFg&_bwcHm;8(&*-a77(J%Ryp^I4Pxa7uEx`GkWmDj3xJQ@m@Hp+AQHqq$+;H{Ktc&f=1ryIL2gUY`Xk3SQ+pv!zy>cCImXfa#5 zwd7Pz&zH2@PN`g2T+oazBiWVE;<=^5&Air|=(cdtqcO(uFAqu)rSjI8bC c<{aBj898SzX`E7)aa0?B?5fDv1b%#f12k*S-T(jq literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-9a39d079-dfcb-46f0-8b5b-ef1e35e78711-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-9a39d079-dfcb-46f0-8b5b-ef1e35e78711-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..41d1f780e804eb82af5ad8ccd4509c67c32a9a3b GIT binary patch literal 1885 zcmc&#ZA@EL7(Vy*w!J`GO4vE+O>SoQgBzA9(js5lD6FeOr^wi9YC=*=xdx?A#>q}S!yK2KQpo0Q1N^9eS6#9y7EJRdD3&w zJ?}Z^dEVza=iGL-cDX2_YdoFj=o^15Em>t6gn&pzLaNW){TFGQfru~2eH_l-E0PMy8) zmmjXqMa~VKz4zy9S7yD3Q}fN0gd+r4;zl~n(WCR{f3pGkU??+YHvQz56y37}B4JOnD!*^}2UlwuaRA}`(!bAmd~!s_zrlG}0k|AO;B zc`eXR+D}h@{L{N858bBQ(88ew1SNnq#0b>{74&C{8J;Cj4Nx_!TEL;Jul@LrO|#0R zTLlvkCddg^yZZneJ!X2{OGnh>6E8L&-Qjg~xits@wxfy8Of62>Jm7VHLyl+}^u~sx zn90+s8r2wa`uO^{pOqIe!V-;P@l?t5{Hyr9%plAEf(>dpWdkkamaLX6KKQG{P8pTU0>I)S)Sd{d_qK$cN7#4)ESe5`Rp9rU@n0K$ z1x-llX!U|Gy~|W6c_1ku+>PJsIRljxBOZ zyi{2)ksCla-Xe#9SE?8d?*U;il+*1jfq-nq&Vq`_-YVM$Uw?lz9DDYlI(pY&%=w%e z8yOfLa=L0<4K=mS>YjrG&-6O)sdY!{nq0M!x<+@eySbsZPIWc88XH~Vh}zSqMk0-_ Y9<{H*wNp9-hr0Mjj=|@$6aHBL21u=amjD0& literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-c59ddaa8-a1c3-4b09-b4a4-9f58a1accd7f-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-0/data-c59ddaa8-a1c3-4b09-b4a4-9f58a1accd7f-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..53d3e4eaa2b734b7daba43fafd90d67be77f736d GIT binary patch literal 1914 zcmc&#Urbw796tAQ+un|r(#Y?mH@PnLp*PG7)|LU=EN1*EGPWUO)XY9uOM9m{St}Hu zNL;2)9~Ndb(}?xYoXK2RM8PFNCkSYF7Z`)ff4}0^Z z=bm$Z_nhzdecw6fwzILz0|b8MVUmN#3oJo#~N%+%@^mT-8`IJ>xnhr>7r`C08%Q$c>|MT1RV zS^nvpfm@$7Jo4vn3!?`=tZvx#$1n5m5snbdbKL+*4z9g<>vz5=aRpEK)-tQ}=Of1T z_4QErbZ|fS(p>xOo|(2Qk4?9J@4wvgUGt@;sm6=DC&#{=yZPMs;^;*C`|W4nxqfbZ zM&8=ZlM;@l5m1g3mY3(x3qIgM$SHG@OwpJKAj9mvMVZlH3-bEQaZ*r+nYuAEX8tb> zk8|!9=6`crfI|?6Z>Lvld_Z*J!X%(G#DLX^74?%tSWQ^XSXn8-s>$4qSSKeI z?uwdSM%^x)z@UPhU@iA{)5NRhGtDq?`lGY2?Rw>*W=EIToPiLa4m#l7tK&Z!ePGnt zENU#G{joS>b3|3+jKN?$8cwKDE&0RH1ZPqozr__L|5VcvoG9*)Y?ZDUw61%61E0Ga_siYYM|mCx%&57~VKX4upAHIGZVz9Hy+m z>>HVikmisckz~5Y!F;YiEI?Z}oHXRSDT6+HNrV%**{MwsXBz2kp5;8*W{rn?`(j5! z1CeLdzA%(w)I}Vo&|d$Oq26fV7#0!l3&zSr2mJ>E{*GpUs3Wky$=?;YhnLW%Ehr5= z(YP<*5AEI4(c;=%yT;zg7akDp8Ev?lk0%05ChHv{PuS3eq%}{nRuGL+cE2--OB@~Hl_gPGFdT8}v(uA7{+y#JPDyQ2R?Ko8smxs@hrljYF(A7NgnnUz&GfDn zwiEh>ody+=UM;P_zlHXAxc`MgHBr^q?|f11ALxx8b$Y5jb=7WXWzS&m^HJw^w>MI= z!{d(B)O(}eoptUS)w9D>U+)P=)Skm?BvS9`Q4iO7wn#&Gu8V)gPPO zkDq_+oco(yGdsWD5v@C0JALBJ{K_+@uKea~W08)b4mqG{!#8fkKIUAQ#axQ94Jl2H ziNhj(d2;2Lfks46YzSk4U9yG~U794H8<${xHQk9~f1(xggf(4@05%2t`-%$CYpcd?`PeotYU z!!r;$QV`LNIJ|{{Bl1W<71xGq@Yv|gJ9#e9eF*Wj}!=%G<`QE1|XDI(kv1l53>*(5=1h};%iKDKFQ&RxA~yH981 z;nePNBMwo5WLifZzTwH0J0o2^-TuDd!Eje6&I4-+vS#@k-Fy9!kZ%wg50?dO7JsX| z+2?NB>Gn7I_Uv%C`0geW3A1{@=x?vv?Q{DZ>zj6!6jk4$Z^X~G@c9ZcIL*raKFX0! z0hgsy)~fFd_Vk8*9Kt48>H0gLEQG`=O7DVBYghU9lga=7eF<}#t$-0gVvl8!8 zv!I!s%KNWC;TmZJ5hY1-n;c`R~ITbO3kpexDMXoW@DC^SFZr!K_@Uc-0 zn)TnluIFJZnK0o;6<-iN#|zeUffP6U$y{1GMac1Tdik z(gN^B>*Yp5Oi%@-1SZ%c_?7I~9cy3anr09^VG`)`+?pg4YW)NXP zL#yU^gOJR(f_R-FW2|&KTl#>9Na+R81khHSMmkHe@1(Fzf{IP6M?rp@AyGyeH%ieQ z(E+^aErL3ME0u{x?*Sq|nsE{NC=aNUr7c93cyiZR03U<~xwH5AfpGtWUA?v!!o7Ww zo^G4H-0mo^u$2Y|BF}|v4_8zMt7_~O!768EsB)X5qAF~!u{)ji&R{rjARG)j?Sb$C UhrLWV1Vx?sL$82;FMNmp0h`mCB>(^b literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f7ead75f-edd3-4307-9473-f5826baf5d96-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f7ead75f-edd3-4307-9473-f5826baf5d96-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..74c4cc1323ab3e6dd7e34f9359951e34ae49f37f GIT binary patch literal 1932 zcmd5-ZA@EL7(Vy*wp?hTCF2}=lN(kNxk92yL0T3Y2ivLzHY^{-EMa!^W3DQ2gPPO zkDq_+oco(yGdsWD5v@C0JALBJ{K_+@uKea~W08)b4mqG{!#8fkKIUAQ#axQ94Jl2H ziNhj(d2;2Lfks46YzSk4U9yG~U794H8<${xHQk9~f1(xggf(4@05%2t`-%$CYpcd?`PeotYU z!!r;$QV`LNIJ|{{Bl1W<71xGq@Yv|gJ9#e9eF*Wj}!=%G<`QE1|XDI(kv1l53>*(5=1h};%iKDKFQ&RxA~yH981 z;nePNBMwo5WLifZzTwH0J0o2^-TuDd!Eje6&I4-+vS#@k-Fy9!kZ%wg50?dO7JsX| z+2?NB>Gn7I_Uv%C`0geW3A1{@=x?vv?Q{DZ>zj6!6jk4$Z^X~G@c9ZcIL*raKFX0! z0hgsy)~fFd_Vk8*9Kt48>H0gLEQG`=O7DVBYghU9lga=7eF<}#t$-0gVvl8!8 zv!I!s%KNWC;TmZJ5hY1-n;c`R~ITbO3kpexDMXoW@DC^SFZr!K_@Uc-0 zn)TnluIFJZnK0o;6<-iN#|zeUffP6U$y{1GMac1Tdik z(gN^B>*Yp5Oi%@-1SZ%c_?7I~9cy3anr09^VG`)`+?pg4YW)NXP zL#yU^gOJR(f_R-FW2|&KTl#>9Na+R81khHSMmkHe@1(Fzf{IP6M?rp@AyGyeH%ieQ z(E+^aErL3ME0u{x?*Sq|nsE{NC=aNUr7c93cyiZR03U<~xwH5AfpGtWUA?v!!o7Ww zo^G4H-0mo^u$2Y|BF}|v4_8zMt7_~O!768EsB)X5qAF~!u{)ji&R{rjARG)j?Sb$C UhrLWV1Vx?sL$82;FMNmp0h`mCB>(^b literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f8e3e01a-3620-40bf-9da7-e20669696822-0.parquet b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/pt=2/bucket-1/data-f8e3e01a-3620-40bf-9da7-e20669696822-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..74c4cc1323ab3e6dd7e34f9359951e34ae49f37f GIT binary patch literal 1932 zcmd5-ZA@EL7(Vy*wp?hTCF2}=lN(kNxk92yL0T3Y2ivLzHY^{-EMa!^W3DQ2gPPO zkDq_+oco(yGdsWD5v@C0JALBJ{K_+@uKea~W08)b4mqG{!#8fkKIUAQ#axQ94Jl2H ziNhj(d2;2Lfks46YzSk4U9yG~U794H8<${xHQk9~f1(xggf(4@05%2t`-%$CYpcd?`PeotYU z!!r;$QV`LNIJ|{{Bl1W<71xGq@Yv|gJ9#e9eF*Wj}!=%G<`QE1|XDI(kv1l53>*(5=1h};%iKDKFQ&RxA~yH981 z;nePNBMwo5WLifZzTwH0J0o2^-TuDd!Eje6&I4-+vS#@k-Fy9!kZ%wg50?dO7JsX| z+2?NB>Gn7I_Uv%C`0geW3A1{@=x?vv?Q{DZ>zj6!6jk4$Z^X~G@c9ZcIL*raKFX0! z0hgsy)~fFd_Vk8*9Kt48>H0gLEQG`=O7DVBYghU9lga=7eF<}#t$-0gVvl8!8 zv!I!s%KNWC;TmZJ5hY1-n;c`R~ITbO3kpexDMXoW@DC^SFZr!K_@Uc-0 zn)TnluIFJZnK0o;6<-iN#|zeUffP6U$y{1GMac1Tdik z(gN^B>*Yp5Oi%@-1SZ%c_?7I~9cy3anr09^VG`)`+?pg4YW)NXP zL#yU^gOJR(f_R-FW2|&KTl#>9Na+R81khHSMmkHe@1(Fzf{IP6M?rp@AyGyeH%ieQ z(E+^aErL3ME0u{x?*Sq|nsE{NC=aNUr7c93cyiZR03U<~xwH5AfpGtWUA?v!!o7Ww zo^G4H-0mo^u$2Y|BF}|v4_8zMt7_~O!768EsB)X5qAF~!u{)ji&R{rjARG)j?Sb$C UhrLWV1Vx?sL$82;FMNmp0h`mCB>(^b literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 new file mode 100644 index 000000000..ddc738c05 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/schema/schema-0 @@ -0,0 +1,36 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "pt", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 2, + "name" : "score", + "type" : "INT" + }, { + "id" : 3, + "name" : "tag", + "type" : "STRING" + } ], + "highestFieldId" : 3, + "partitionKeys" : [ "pt" ], + "primaryKeys" : [ "pt", "id" ], + "options" : { + "bucket" : "2", + "compaction.force-rewrite-all-files" : "true", + "target-file-size" : "8 kb", + "write-only" : "true", + "num-sorted-run.compaction-trigger" : "10000", + "deletion-vectors.merge-on-read" : "false", + "pk-btree.index.columns" : "score", + "file.format" : "parquet", + "deletion-vectors.enabled" : "true" + }, + "timeMillis" : 1786866468700 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST new file mode 100644 index 000000000..56a6051ca --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST new file mode 100644 index 000000000..7813681f5 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/LATEST @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 new file mode 100644 index 000000000..2f1c2804b --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "uuid" : "420b22a8-a0e0-4818-adb2-655c492dec96", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-bcb0c671-cdf9-4025-9b89-23e9ba256c9a-1", + "deltaManifestListSize" : 1119, + "commitUser" : "711e6dbd-2af8-4b27-9793-1cfca17d9c00", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866468750, + "totalRecordCount" : 200, + "deltaRecordCount" : 200, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 new file mode 100644 index 000000000..c4ef63281 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-2 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "1f5e235e-3e28-4ab4-812f-6a1e4ca39056", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "b2b00e43-1a8e-4cb1-84da-0f003902e88b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468867, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 new file mode 100644 index 000000000..8d40875fc --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-3 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "fa35fcd9-4728-4e75-8e13-d0291a13cb3d", + "id" : 3, + "schemaId" : 0, + "baseManifestList" : "manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-0", + "baseManifestListSize" : 1158, + "deltaManifestList" : "manifest-list-692295fa-2039-44ff-b207-b0df5c7f715b-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "49a0d61e-b7ad-44ac-8a54-d9de191cb80e", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1786866468901, + "totalRecordCount" : 202, + "deltaRecordCount" : 2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 new file mode 100644 index 000000000..33f16fd31 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-4 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "4e6c3373-056b-4624-8191-1a38f9ba4b23", + "id" : 4, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-0", + "baseManifestListSize" : 1196, + "deltaManifestList" : "manifest-list-1943462b-63ba-44d3-8893-0f2557f3b1a9-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-a9b54c7b-7468-4a64-97ee-32adae28681e-0", + "commitUser" : "01221812-aecb-4b28-8128-d081617d48a0", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468941, + "totalRecordCount" : 203, + "deltaRecordCount" : 1, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 new file mode 100644 index 000000000..c7926bc1d --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/snapshot/snapshot-5 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "35a01751-18a9-43d7-8733-4a3ebd1b94c1", + "id" : 5, + "schemaId" : 0, + "baseManifestList" : "manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-0", + "baseManifestListSize" : 1234, + "deltaManifestList" : "manifest-list-f1757963-61f0-4136-bae6-cf52e0b2f42c-1", + "deltaManifestListSize" : 1126, + "indexManifest" : "index-manifest-11be7772-670d-4dc1-adbb-fd8413f2779d-0", + "commitUser" : "8e5448ed-d149-4e78-a7da-10d9616d70b3", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468999, + "totalRecordCount" : 201, + "deltaRecordCount" : -2, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base new file mode 100644 index 000000000..c4ef63281 --- /dev/null +++ b/test/test_data/parquet/pk_btree_partitioned_e2e.db/pk_btree_partitioned_e2e/tag/tag-fallback-base @@ -0,0 +1,18 @@ +{ + "version" : 3, + "uuid" : "1f5e235e-3e28-4ab4-812f-6a1e4ca39056", + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-1b95a9b1-e06a-41bf-afb8-07cd03eb8ae1-1", + "deltaManifestListSize" : 1123, + "indexManifest" : "index-manifest-74eb4325-f527-47fc-aeea-25b0928190f7-0", + "commitUser" : "b2b00e43-1a8e-4cb1-84da-0f003902e88b", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "COMPACT", + "timeMillis" : 1786866468867, + "totalRecordCount" : 200, + "deltaRecordCount" : 0, + "nextRowId" : 0 +} \ No newline at end of file From 1e4c9e2a5dcbdca0b8c001fc7026f2782905b04c Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Sun, 16 Aug 2026 22:47:00 -0400 Subject: [PATCH 11/14] fix(pk-index): correct executor and fallback handling --- .../btree/btree_global_indexer.cpp | 6 +++- .../union_global_index_reader_test.cpp | 8 ++++- .../source/primary_key_index_batch_scan.cpp | 32 ++++--------------- .../source/primary_key_sorted_index_scan.cpp | 14 +++++--- .../source/primary_key_sorted_index_scan.h | 11 +++---- .../primary_key_sorted_index_scan_test.cpp | 3 +- 6 files changed, 35 insertions(+), 39 deletions(-) diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index 74bdfde10..dad0c8810 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -39,6 +39,7 @@ #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/preconditions.h" #include "paimon/core/options/compress_options.h" +#include "paimon/executor.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/memory/bytes.h" #include "paimon/utils/roaring_bitmap64.h" @@ -105,7 +106,10 @@ Result> BTreeGlobalIndexer::CreateWriter( Result> BTreeGlobalIndexer::CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, const std::vector& files, const std::shared_ptr& pool) const { - return CreateReader(arrow_schema, file_reader, files, pool, /*executor=*/nullptr); + // Preserve the compatibility overload's existing private executor. Scan paths which can + // safely share an executor use the overload below. + std::shared_ptr executor = CreateDefaultExecutor(); + return CreateReader(arrow_schema, file_reader, files, pool, executor); } Result> BTreeGlobalIndexer::CreateReader( diff --git a/src/paimon/common/global_index/union_global_index_reader_test.cpp b/src/paimon/common/global_index/union_global_index_reader_test.cpp index 6c3c68585..721255264 100644 --- a/src/paimon/common/global_index/union_global_index_reader_test.cpp +++ b/src/paimon/common/global_index/union_global_index_reader_test.cpp @@ -220,6 +220,10 @@ class DeferAfterFirstExecutor : public Executor { return 1; } + uint32_t SubmissionCount() const { + return submission_count_; + } + void RunPendingTasks() { while (!pending_tasks_.empty()) { std::function task = std::move(pending_tasks_.front()); @@ -265,13 +269,15 @@ class UnionGlobalIndexReaderTest : public ::testing::Test { TEST_F(UnionGlobalIndexReaderTest, TestSingleReaderUnion) { auto reader = std::make_shared(); reader->SetDefaultResult({1, 2, 3}); + auto executor = std::make_shared(); std::vector> readers = {reader}; - UnionGlobalIndexReader union_reader(std::move(readers), nullptr); + UnionGlobalIndexReader union_reader(std::move(readers), executor); ASSERT_OK_AND_ASSIGN(auto result, union_reader.VisitIsNotNull()); CheckResult(result, {1, 2, 3}); ASSERT_EQ(reader->InvocationCount(), 1); + ASSERT_EQ(executor->SubmissionCount(), 0); } TEST_F(UnionGlobalIndexReaderTest, TestMultipleReadersUnionSequential) { diff --git a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp index e65b6aa30..7e3872d23 100644 --- a/src/paimon/core/table/source/primary_key_index_batch_scan.cpp +++ b/src/paimon/core/table/source/primary_key_index_batch_scan.cpp @@ -24,9 +24,7 @@ #include #include #include -#include -#include "fmt/format.h" #include "paimon/core/index/index_file_handler.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" #include "paimon/core/table/source/plan_impl.h" @@ -35,29 +33,10 @@ #include "paimon/core/table/source/snapshot/snapshot_reader.h" #include "paimon/core/utils/index_file_path_factories.h" #include "paimon/core/utils/snapshot_manager.h" -#include "paimon/executor.h" +#include "paimon/logging.h" #include "paimon/predicate/predicate_utils.h" namespace paimon { -namespace { -Result> CreateGlobalIndexExecutor(const CoreOptions& core_options) { - uint32_t thread_num = std::thread::hardware_concurrency(); - std::optional configured_thread_num = core_options.GetGlobalIndexThreadNum(); - if (configured_thread_num) { - if (configured_thread_num.value() <= 0) { - return Status::Invalid(fmt::format("invalid global index thread number {}", - configured_thread_num.value())); - } - thread_num = static_cast(configured_thread_num.value()); - } else if (thread_num == 0) { - thread_num = 1; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, CreateDefaultExecutor(thread_num)); - return executor; -} - -} // namespace - Result> PrimaryKeyIndexBatchScan::Create( const std::shared_ptr& snapshot_reader, std::unique_ptr&& batch_scan, @@ -109,6 +88,11 @@ Result> PrimaryKeyIndexBatchScan::CreatePlan() { snapshot_reader_->GetSnapshotManager(); Result snapshot_result = snapshot_manager->LoadSnapshot(snapshot_id); if (!snapshot_result.ok()) { + static auto logger = Logger::GetLogger("PrimaryKeyIndexBatchScan"); + PAIMON_LOG_WARN(logger, + "Failed to load snapshot %ld for primary-key sorted-index planning; " + "falling back to the unindexed data plan: %s", + snapshot_id, snapshot_result.status().ToString().c_str()); return data_plan; } @@ -138,12 +122,10 @@ Result> PrimaryKeyIndexBatchScan::CreatePlan() { if (!has_index_group) { return data_plan; } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, - CreateGlobalIndexExecutor(core_options_)); PrimaryKeySortedIndexScan::ReaderFactory reader_factory = PrimaryKeySortedIndexScan::MakeReaderFactory( core_options_.GetFileSystem(), std::make_shared(path_factory_), - table_schema_, pool_, executor); + table_schema_, pool_); PAIMON_ASSIGN_OR_RAISE( PrimaryKeySortedIndexScan::EvaluatedPlan evaluated_plan, PrimaryKeySortedIndexScan::Evaluate(index_plan, table_schema_, predicate, diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp index e2af0a2c8..26028cc4f 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -40,6 +40,7 @@ #include "paimon/global_index/global_indexer.h" #include "paimon/global_index/global_indexer_factory.h" #include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/logging.h" #include "paimon/predicate/predicate_utils.h" namespace paimon { @@ -475,6 +476,11 @@ Result PrimaryKeySortedIndexScan::Eval files.emplace_back(file, result.value()); } else { // Evaluation failures degrade to a normal scan for this file only. + static auto logger = Logger::GetLogger("PrimaryKeySortedIndexScan"); + PAIMON_LOG_WARN(logger, + "Failed to evaluate primary-key sorted index for data file %s; " + "falling back to a normal scan for this file: %s", + file.DataFile()->file_name.c_str(), result.status().ToString().c_str()); files.emplace_back(file, nullptr); } } @@ -500,10 +506,9 @@ class FsGlobalIndexFileReader : public GlobalIndexFileReader { PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFactory( const std::shared_ptr& file_system, const std::shared_ptr& path_factories, - const std::shared_ptr& table_schema, const std::shared_ptr& pool, - const std::shared_ptr& executor) { + const std::shared_ptr& table_schema, const std::shared_ptr& pool) { auto file_reader = std::make_shared(file_system); - return [path_factories, table_schema, pool, file_reader, executor]( + return [path_factories, table_schema, pool, file_reader]( const FilePlan& file, const PrimaryKeyIndexDefinition& definition, const PkSortedIndexGroup& group) -> Result> { if (definition.GetFamily() != PrimaryKeyIndexDefinition::Family::BTREE) { @@ -537,7 +542,8 @@ PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFa ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); - return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool, executor); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool, + /*executor=*/nullptr); }; } diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.h b/src/paimon/core/table/source/primary_key_sorted_index_scan.h index a892f82ce..6ff73bbdb 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.h +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.h @@ -41,8 +41,6 @@ #include "paimon/result.h" namespace paimon { -class Executor; - /// Plans and evaluates source-backed primary-key scalar index groups in file-local /// row-position space. /// @@ -175,14 +173,13 @@ class PrimaryKeySortedIndexScan { const std::vector& definitions, const ReaderFactory& reader_factory); - /// Creates the default reader factory which opens BTree payloads through the table's - /// index directory layout. Non-BTree families resolve to a null reader and therefore - /// keep normal scan semantics. + /// Creates the default reader factory which opens the single BTree payload of each validated + /// group through the table's index directory layout. Non-BTree families resolve to a null + /// reader and therefore keep normal scan semantics. static ReaderFactory MakeReaderFactory( const std::shared_ptr& file_system, const std::shared_ptr& path_factories, - const std::shared_ptr& table_schema, const std::shared_ptr& pool, - const std::shared_ptr& executor); + const std::shared_ptr& table_schema, const std::shared_ptr& pool); }; } // namespace paimon diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index 5ef555bfe..ea220d6d2 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -295,7 +295,8 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); auto file_reader = std::make_shared(fs); - return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool, + /*executor=*/nullptr); }; } From 6955a870ae8af361c128a849dc2a2545d48fb1d2 Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Mon, 17 Aug 2026 22:00:11 -0400 Subject: [PATCH 12/14] refactor(global-index): avoid executor API expansion --- include/paimon/global_index/global_indexer.h | 23 --------------- .../btree_global_index_integration_test.cpp | 29 ++----------------- .../btree/btree_global_indexer.cpp | 16 ++++------ .../global_index/btree/btree_global_indexer.h | 5 ---- .../source/primary_key_sorted_index_scan.cpp | 3 +- .../primary_key_sorted_index_scan_test.cpp | 3 +- 6 files changed, 10 insertions(+), 69 deletions(-) diff --git a/include/paimon/global_index/global_indexer.h b/include/paimon/global_index/global_indexer.h index 690ec4bd1..4da6293ff 100644 --- a/include/paimon/global_index/global_indexer.h +++ b/include/paimon/global_index/global_indexer.h @@ -35,8 +35,6 @@ struct ArrowSchema; namespace paimon { -class Executor; - /// Interface for creating global index readers and writers. class PAIMON_EXPORT GlobalIndexer { public: @@ -72,27 +70,6 @@ class PAIMON_EXPORT GlobalIndexer { ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, const std::vector& files, const std::shared_ptr& pool) const = 0; - - /// Creates a reader using an executor supplied by the scan layer. - /// - /// Index implementations which do not perform asynchronous work may ignore the executor and - /// use the compatibility overload above. - /// - /// @param arrow_schema Schema of the indexed data; used to interpret predicate literals. - /// @param file_reader I/O handler for reading index artifacts from storage. - /// @param files List of index file metadata entries produced during writing. - /// @param pool Memory pool for temporary allocations; if nullptr, uses default. - /// @param executor Executor shared by readers created for the same scan; nullptr means - /// that the reader should evaluate sequentially. - /// @return A `Result` containing a shared pointer to the created `GlobalIndexReader`, - /// or an error if the index cannot be loaded or is incompatible, etc. - virtual Result> CreateReader( - ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, const std::shared_ptr& pool, - const std::shared_ptr& executor) const { - static_cast(executor); - return CreateReader(arrow_schema, file_reader, files, pool); - } }; } // namespace paimon diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index 9bfb97e0c..bbc726b89 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -#include #include #include @@ -33,7 +32,6 @@ #include "paimon/common/utils/scope_guard.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" -#include "paimon/executor.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/io/global_index_file_reader.h" @@ -89,27 +87,6 @@ class FakeGlobalIndexFileReader : public GlobalIndexFileReader { std::string base_path_; }; -class CountingInlineExecutor : public Executor { - public: - void Add(std::function func) override { - submission_count_.fetch_add(1); - func(); - } - - void ShutdownNow() override {} - - uint32_t GetThreadNum() const override { - return 1; - } - - uint32_t SubmissionCount() const { - return submission_count_.load(); - } - - private: - std::atomic submission_count_{0}; -}; - class BTreeGlobalIndexIntegrationTest : public ::testing::Test, public ::testing::WithParamInterface { protected: @@ -1998,10 +1975,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadMultiFilesWithMetaSelector) // Create reader over all 3 files (internally uses LazyFilteredBTreeReader + // BTreeFileMetaSelector) auto file_reader = std::make_shared(fs_, base_path_); - auto executor = std::make_shared(); auto c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, all_metas, - pool_, executor)); + ASSERT_OK_AND_ASSIGN(auto reader, + indexer->CreateReader(c_schema.get(), file_reader, all_metas, pool_)); // --- VisitEqual: key=12 -> only file1 is selected by meta selector -> row 5 { @@ -2050,7 +2026,6 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadMultiFilesWithMetaSelector) Literal literal_5(5); ASSERT_OK_AND_ASSIGN(auto result, reader->VisitGreaterOrEqual(literal_5)); CheckResult(result, {3, 4, 5, 6, 7, 9}); - ASSERT_EQ(executor->SubmissionCount(), 3); } // --- VisitLessOrEqual: key <= 2 -> only file0 selected -> rows 0,1 diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index dad0c8810..36176fa7f 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -106,16 +106,6 @@ Result> BTreeGlobalIndexer::CreateWriter( Result> BTreeGlobalIndexer::CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, const std::vector& files, const std::shared_ptr& pool) const { - // Preserve the compatibility overload's existing private executor. Scan paths which can - // safely share an executor use the overload below. - std::shared_ptr executor = CreateDefaultExecutor(); - return CreateReader(arrow_schema, file_reader, files, pool, executor); -} - -Result> BTreeGlobalIndexer::CreateReader( - ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, const std::shared_ptr& pool, - const std::shared_ptr& executor) const { // Get field type from arrow schema PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, arrow::ImportSchema(arrow_schema)); @@ -136,6 +126,12 @@ Result> BTreeGlobalIndexer::CreateReader( } read_buffer_size = static_cast(tmp_buffer_size); } + // UnionGlobalIndexReader evaluates one payload inline. Preserve the existing private pool only + // when multiple payloads can submit work. + std::shared_ptr executor; + if (files.size() > 1) { + executor = CreateDefaultExecutor(); + } return std::make_shared(read_buffer_size, files, key_type, file_reader, cache_manager_, pool, executor); } diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.h b/src/paimon/common/global_index/btree/btree_global_indexer.h index f93099487..5568adba3 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.h +++ b/src/paimon/common/global_index/btree/btree_global_indexer.h @@ -68,11 +68,6 @@ class BTreeGlobalIndexer : public GlobalIndexer { const std::vector& files, const std::shared_ptr& pool) const override; - Result> CreateReader( - ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, const std::shared_ptr& pool, - const std::shared_ptr& executor) const override; - private: BTreeGlobalIndexer(const std::shared_ptr& cache_manager, const std::map& options) diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp index 26028cc4f..325288e2b 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -542,8 +542,7 @@ PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFa ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); - return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool, - /*executor=*/nullptr); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); }; } diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index ea220d6d2..5ef555bfe 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -295,8 +295,7 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); auto file_reader = std::make_shared(fs); - return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool, - /*executor=*/nullptr); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); }; } From 01a145df451b03f10a54af65558f951ff1e4b2b7 Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Tue, 18 Aug 2026 22:23:03 -0400 Subject: [PATCH 13/14] fix(pk-index): adapt to FileStatus value API --- .../core/table/source/primary_key_sorted_index_scan_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index 5ef555bfe..e75591cd7 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -67,9 +67,9 @@ class TestGlobalIndexFileWriter : public GlobalIndexFileWriter { } Result GetFileSize(const std::string& file_name) const override { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_status, + PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs_->GetFileStatus(base_path_ + "/" + file_name)); - return file_status->GetLen(); + return file_status.GetLen(); } std::string ToPath(const std::string& file_name) const override { From 30ea746920082859e2e9c477aad5e44028a16ce0 Mon Sep 17 00:00:00 2001 From: "wangyong.alen" Date: Wed, 19 Aug 2026 01:25:50 -0400 Subject: [PATCH 14/14] fix(pk-index): harden source-backed reads --- .../btree/btree_compatibility_test.cpp | 15 +- .../btree/btree_file_meta_selector.cpp | 70 +++++++-- .../btree/btree_file_meta_selector.h | 10 +- .../btree/btree_file_meta_selector_test.cpp | 129 +++++++++++----- .../btree_global_index_integration_test.cpp | 4 +- .../btree/btree_global_index_reader.cpp | 2 + .../btree/btree_global_indexer.cpp | 7 +- .../global_index/btree/btree_index_meta.cpp | 89 +++++++++-- .../global_index/btree/btree_index_meta.h | 5 +- .../btree/btree_index_meta_test.cpp | 106 ++++++++++++-- .../global_index/btree/key_serializer.cpp | 138 ++++++++++++++++++ .../global_index/btree/key_serializer.h | 3 + .../btree/key_serializer_test.cpp | 36 +++++ .../btree/lazy_filtered_btree_reader.cpp | 66 +++++---- .../btree/lazy_filtered_btree_reader.h | 22 ++- .../btree/lazy_filtered_btree_reader_test.cpp | 50 +++++-- .../core/operation/raw_file_split_read.cpp | 15 +- .../operation/raw_file_split_read_test.cpp | 36 ++++- .../table/source/key_value_table_read.cpp | 18 ++- .../source/primary_key_sorted_index_scan.cpp | 16 +- .../primary_key_sorted_index_scan_test.cpp | 89 ++++++++++- 21 files changed, 779 insertions(+), 147 deletions(-) diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp index c76b3ea89..84b451758 100644 --- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp +++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp @@ -783,7 +783,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -808,7 +809,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -833,7 +835,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); @@ -848,7 +851,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->FirstKey()); @@ -870,7 +874,8 @@ TEST_F(BTreeCompatibilityTest, MetaDeserialization) { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); - auto meta = BTreeIndexMeta::Deserialize(meta_bytes, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(meta_bytes, pool_.get())); ASSERT_TRUE(meta); ASSERT_TRUE(meta->HasNulls()); diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp index cedfa4a73..aa2f23c71 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp @@ -19,22 +19,70 @@ #include "paimon/common/global_index/btree/btree_file_meta_selector.h" +#include "fmt/format.h" #include "paimon/common/memory/memory_slice.h" namespace paimon { -BTreeFileMetaSelector::BTreeFileMetaSelector(const std::vector& files, - const std::shared_ptr& key_type, - const std::shared_ptr& pool) - : key_type_(key_type), - pool_(pool), - comparator_(KeySerializer::CreateComparator(key_type, pool)) { - files_.reserve(files.size()); +Result> BTreeFileMetaSelector::Create( + const std::vector& files, const std::shared_ptr& key_type, + const std::shared_ptr& pool) { + if (key_type == nullptr) { + return Status::Invalid("Cannot create a BTree file metadata selector without a key type."); + } + if (pool == nullptr) { + return Status::Invalid( + "Cannot create a BTree file metadata selector without a memory pool."); + } + std::vector>> decoded_files; + decoded_files.reserve(files.size()); + MemorySlice::SliceComparator comparator = KeySerializer::CreateComparator(key_type, pool); for (const auto& file : files) { - auto index_meta = BTreeIndexMeta::Deserialize(file.metadata, pool.get()); - files_.emplace_back(file, std::move(index_meta)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_meta, + BTreeIndexMeta::Deserialize(file.metadata, pool.get())); + bool has_first_key = index_meta->FirstKey() != nullptr; + bool has_last_key = index_meta->LastKey() != nullptr; + if (has_first_key != has_last_key) { + return Status::Invalid(fmt::format( + "BTree index metadata for {} must contain both boundary keys or neither.", + file.file_path)); + } + if (!has_first_key && !index_meta->HasNulls()) { + return Status::Invalid( + fmt::format("BTree index metadata for {} has no boundary keys or null values.", + file.file_path)); + } + if (index_meta->FirstKey() != nullptr) { + PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey( + WrapKeySlice(index_meta->FirstKey()), key_type)); + } + if (index_meta->LastKey() != nullptr) { + PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey( + WrapKeySlice(index_meta->LastKey()), key_type)); + } + if (index_meta->FirstKey() != nullptr && index_meta->LastKey() != nullptr) { + PAIMON_ASSIGN_OR_RAISE(int32_t comparison, + comparator(WrapKeySlice(index_meta->FirstKey()), + WrapKeySlice(index_meta->LastKey()))); + if (comparison > 0) { + return Status::Invalid(fmt::format( + "BTree index metadata for {} has a first key greater than its last key.", + file.file_path)); + } + } + decoded_files.emplace_back(file, std::move(index_meta)); } + return std::unique_ptr( + new BTreeFileMetaSelector(std::move(decoded_files), key_type, pool)); } +BTreeFileMetaSelector::BTreeFileMetaSelector( + std::vector>> files, + std::shared_ptr key_type, std::shared_ptr pool) + : files_(std::move(files)), + key_type_(std::move(key_type)), + pool_(std::move(pool)), + comparator_(KeySerializer::CreateComparator(key_type_, pool_)) {} + Result> BTreeFileMetaSelector::VisitIsNotNull() { return Filter([](const BTreeIndexMeta& meta) -> Result { return !meta.OnlyNulls(); }); } @@ -208,7 +256,9 @@ MemorySlice BTreeFileMetaSelector::WrapKeySlice(const std::shared_ptr& ke Result BTreeFileMetaSelector::SerializeLiteral(const Literal& literal) const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bytes, KeySerializer::SerializeKey(literal, key_type_, pool_.get())); - return MemorySlice::Wrap(bytes); + MemorySlice slice = MemorySlice::Wrap(bytes); + PAIMON_RETURN_NOT_OK(KeySerializer::ValidateSerializedKey(slice, key_type_)); + return slice; } } // namespace paimon diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.h b/src/paimon/common/global_index/btree/btree_file_meta_selector.h index b59fc7ed7..ad066278e 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.h +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector.h @@ -34,9 +34,9 @@ namespace paimon { /// Selects candidate BTree index files based on filter predicates. class BTreeFileMetaSelector : public FunctionVisitor> { public: - BTreeFileMetaSelector(const std::vector& files, - const std::shared_ptr& key_type, - const std::shared_ptr& pool); + static Result> Create( + const std::vector& files, + const std::shared_ptr& key_type, const std::shared_ptr& pool); Result> VisitIsNotNull() override; Result> VisitIsNull() override; @@ -55,6 +55,10 @@ class BTreeFileMetaSelector : public FunctionVisitor> VisitLike(const Literal& literal) override; private: + BTreeFileMetaSelector( + std::vector>> files, + std::shared_ptr key_type, std::shared_ptr pool); + using MetaPredicate = std::function(const BTreeIndexMeta&)>; Result> Filter(const MetaPredicate& predicate) const; diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp index ce77207bd..bb025c118 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp @@ -87,99 +87,111 @@ class BTreeFileMetaSelectorTest : public ::testing::Test { }; TEST_F(BTreeFileMetaSelectorTest, TestVisitLessThan) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // minKey < 8: file1(1), file4(1) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitLessThan(Literal(8))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitLessThan(Literal(8))); CheckResult(result, {"file1", "file4"}); // minKey < 15: file1(1), file4(1) (file2 minKey=15, not < 15) - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(15))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessThan(Literal(15))); CheckResult(result, {"file1", "file4"}); // minKey < 1: no file has minKey < 1 - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(1))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessThan(Literal(1))); ASSERT_TRUE(result.empty()); } TEST_F(BTreeFileMetaSelectorTest, TestVisitLessOrEqual) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // minKey <= 20: file1(1), file2(15), file4(1), file5(19) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitLessOrEqual(Literal(20))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitLessOrEqual(Literal(20))); CheckResult(result, {"file1", "file2", "file4", "file5"}); // minKey <= 15: file1(1), file2(15), file4(1) - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessOrEqual(Literal(15))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessOrEqual(Literal(15))); CheckResult(result, {"file1", "file2", "file4"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterThan) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // maxKey > 20: file3(30), file5(25) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitGreaterThan(Literal(20))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitGreaterThan(Literal(20))); CheckResult(result, {"file3", "file5"}); // maxKey > 30: no file - ASSERT_OK_AND_ASSIGN(result, selector.VisitGreaterThan(Literal(30))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitGreaterThan(Literal(30))); ASSERT_TRUE(result.empty()); } TEST_F(BTreeFileMetaSelectorTest, TestVisitGreaterOrEqual) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // maxKey >= 5: all non-null files (file1..file5) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitGreaterOrEqual(Literal(5))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitGreaterOrEqual(Literal(5))); CheckResult(result, {"file1", "file2", "file3", "file4", "file5"}); // maxKey >= 20: file2(20), file3(30), file5(25) - ASSERT_OK_AND_ASSIGN(result, selector.VisitGreaterOrEqual(Literal(20))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitGreaterOrEqual(Literal(20))); CheckResult(result, {"file2", "file3", "file5"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitEqual) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // 22 in [21,30] and [19,25] - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitEqual(Literal(22))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitEqual(Literal(22))); CheckResult(result, {"file3", "file5"}); // 30 in [21,30] only - ASSERT_OK_AND_ASSIGN(result, selector.VisitEqual(Literal(30))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitEqual(Literal(30))); CheckResult(result, {"file3"}); // 100 out of all ranges - ASSERT_OK_AND_ASSIGN(result, selector.VisitEqual(Literal(100))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitEqual(Literal(100))); ASSERT_TRUE(result.empty()); + + // A mismatched literal must fail before the fixed-width comparator reads it. + ASSERT_NOK(selector->VisitEqual(Literal(static_cast(1)))); } TEST_F(BTreeFileMetaSelectorTest, TestVisitNotEqual) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // NotEqual cannot prune any file, returns all - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitNotEqual(Literal(22))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitNotEqual(Literal(22))); CheckResult(result, {"file1", "file2", "file3", "file4", "file5", "file6"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitIsNull) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // has_nulls: file1, file3, file5, file6 - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitIsNull()); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIsNull()); CheckResult(result, {"file1", "file3", "file5", "file6"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitIsNotNull) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // !onlyNulls: file1..file5 (file6 is only-nulls) - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitIsNotNull()); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIsNotNull()); CheckResult(result, {"file1", "file2", "file3", "file4", "file5"}); } TEST_F(BTreeFileMetaSelectorTest, TestVisitIn) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // IN(1, 2, 3, 26, 27, 28): // 1 in [1,10]=file1, [1,5]=file4 @@ -188,42 +200,44 @@ TEST_F(BTreeFileMetaSelectorTest, TestVisitIn) { // 26 in [21,30]=file3 // 27 in [21,30]=file3 // 28 in [21,30]=file3 - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitIn({Literal(1), Literal(2), Literal(3), - Literal(26), Literal(27), Literal(28)})); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitIn({Literal(1), Literal(2), Literal(3), + Literal(26), Literal(27), Literal(28)})); CheckResult(result, {"file1", "file3", "file4"}); // IN(100): no match - ASSERT_OK_AND_ASSIGN(result, selector.VisitIn({Literal(100)})); + ASSERT_OK_AND_ASSIGN(result, selector->VisitIn({Literal(100)})); ASSERT_TRUE(result.empty()); } TEST_F(BTreeFileMetaSelectorTest, TestVisitNotIn) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // NotIn cannot prune any file ASSERT_OK_AND_ASSIGN(auto result, - selector.VisitNotIn({Literal(1), Literal(7), Literal(19), Literal(30)})); + selector->VisitNotIn({Literal(1), Literal(7), Literal(19), Literal(30)})); CheckResult(result, {"file1", "file2", "file3", "file4", "file5", "file6"}); } TEST_F(BTreeFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) { - BTreeFileMetaSelector selector(files_, key_type_, pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files_, key_type_, pool_)); // file6 is only-nulls, should be excluded from all range/equality queries - ASSERT_OK_AND_ASSIGN(auto result, selector.VisitEqual(Literal(1))); + ASSERT_OK_AND_ASSIGN(auto result, selector->VisitEqual(Literal(1))); auto names = FileNames(result); ASSERT_EQ(names.count("file6"), 0u); - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(100))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessThan(Literal(100))); names = FileNames(result); ASSERT_EQ(names.count("file6"), 0u); - ASSERT_OK_AND_ASSIGN(result, selector.VisitGreaterThan(Literal(0))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitGreaterThan(Literal(0))); names = FileNames(result); ASSERT_EQ(names.count("file6"), 0u); // But IsNull should include file6 - ASSERT_OK_AND_ASSIGN(result, selector.VisitIsNull()); + ASSERT_OK_AND_ASSIGN(result, selector->VisitIsNull()); names = FileNames(result); ASSERT_EQ(names.count("file6"), 1u); } @@ -249,22 +263,57 @@ TEST_F(BTreeFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) { GlobalIndexIOMeta("file_nulls", 1, null_meta->Serialize(pool.get())), }; - BTreeFileMetaSelector selector(files, key_type, pool); + ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, + BTreeFileMetaSelector::Create(files, key_type, pool)); ASSERT_OK_AND_ASSIGN(std::vector result, - selector.VisitEqual(Literal(FieldType::STRING, "www.example.com", 15))); + selector->VisitEqual(Literal(FieldType::STRING, "www.example.com", 15))); CheckResult(result, {"file_empty", "file_normal"}); - ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(FieldType::STRING, "bbb.com", 7))); + ASSERT_OK_AND_ASSIGN(result, selector->VisitLessThan(Literal(FieldType::STRING, "bbb.com", 7))); CheckResult(result, {"file_empty", "file_normal"}); ASSERT_OK_AND_ASSIGN( - result, selector.VisitGreaterThan(Literal(FieldType::STRING, "www.example.com", 15))); + result, selector->VisitGreaterThan(Literal(FieldType::STRING, "www.example.com", 15))); CheckResult(result, {"file_normal"}); - ASSERT_OK_AND_ASSIGN(result, selector.VisitIn({Literal(FieldType::STRING, "", 0), - Literal(FieldType::STRING, "zzz.com", 7)})); + ASSERT_OK_AND_ASSIGN(result, selector->VisitIn({Literal(FieldType::STRING, "", 0), + Literal(FieldType::STRING, "zzz.com", 7)})); CheckResult(result, {"file_empty", "file_normal"}); } +TEST_F(BTreeFileMetaSelectorTest, RejectsMalformedFileMetadata) { + std::shared_ptr pool = GetDefaultPool(); + std::vector files = {GlobalIndexIOMeta("missing", 1, /*metadata=*/nullptr)}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto truncated = std::make_shared(std::string(4, '\0'), pool.get()); + files = {GlobalIndexIOMeta("truncated", 1, truncated)}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto short_key = std::make_shared(std::string(1, '\0'), pool.get()); + auto invalid_key_meta = std::make_shared(short_key, short_key, false); + files = {GlobalIndexIOMeta("invalid-key", 1, invalid_key_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto reversed_meta = std::make_shared(SerializeInt(10), SerializeInt(1), false); + files = {GlobalIndexIOMeta("reversed-range", 1, reversed_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto only_first_meta = + std::make_shared(SerializeInt(1), /*last_key=*/nullptr, false); + files = {GlobalIndexIOMeta("only-first-key", 1, only_first_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto only_last_meta = + std::make_shared(/*first_key=*/nullptr, SerializeInt(1), false); + files = {GlobalIndexIOMeta("only-last-key", 1, only_last_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); + + auto empty_nonnull_meta = + std::make_shared(/*first_key=*/nullptr, /*last_key=*/nullptr, false); + files = {GlobalIndexIOMeta("empty-nonnull", 1, empty_nonnull_meta->Serialize(pool.get()))}; + ASSERT_NOK(BTreeFileMetaSelector::Create(files, arrow::int32(), pool)); +} + } // namespace paimon::test diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index 93bbbe15f..653b617d3 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -490,8 +490,8 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteEmptyStringKeyMetadata) { ASSERT_OK_AND_ASSIGN(auto metas, writer->Finish()); ASSERT_EQ(metas.size(), 1); - std::shared_ptr meta = - BTreeIndexMeta::Deserialize(metas[0].metadata, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr meta, + BTreeIndexMeta::Deserialize(metas[0].metadata, pool_.get())); ASSERT_TRUE(meta->FirstKey()); ASSERT_EQ(meta->FirstKey()->size(), 0); ASSERT_TRUE(meta->LastKey()); diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp index 2ed879875..f9a4868ff 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp @@ -291,6 +291,8 @@ Result BTreeGlobalIndexReader::RangeQuery(const std::optional= from, so skip lower bound comparison. diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index 36176fa7f..8996fe59d 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -132,8 +132,11 @@ Result> BTreeGlobalIndexer::CreateReader( if (files.size() > 1) { executor = CreateDefaultExecutor(); } - return std::make_shared(read_buffer_size, files, key_type, file_reader, - cache_manager_, pool, executor); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr reader, + LazyFilteredBTreeReader::Create(read_buffer_size, files, key_type, file_reader, + cache_manager_, pool, executor)); + return std::shared_ptr(std::move(reader)); } } // namespace paimon diff --git a/src/paimon/common/global_index/btree/btree_index_meta.cpp b/src/paimon/common/global_index/btree/btree_index_meta.cpp index 9ba88e534..4b6f6ce0e 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.cpp +++ b/src/paimon/common/global_index/btree/btree_index_meta.cpp @@ -19,12 +19,28 @@ #include "paimon/common/global_index/btree/btree_index_meta.h" +#include +#include + +#include "fmt/format.h" #include "paimon/common/memory/memory_slice_output.h" namespace paimon { namespace { -std::shared_ptr ReadKey(MemorySliceInput* input, int32_t key_length, MemoryPool* pool) { +Result> ReadKey(MemorySliceInput* input, int32_t key_length, + int32_t required_remaining, const char* key_name, + MemoryPool* pool) { + if (key_length < 0) { + return Status::Invalid( + fmt::format("BTree index metadata has a negative {} length {}.", key_name, key_length)); + } + if (input->Available() < required_remaining || + key_length > input->Available() - required_remaining) { + return Status::Invalid( + fmt::format("BTree index metadata {} length {} exceeds the available payload bytes.", + key_name, key_length)); + } if (key_length == 0) { return std::make_shared(0, pool); } @@ -33,32 +49,77 @@ std::shared_ptr ReadKey(MemorySliceInput* input, int32_t key_length, Memo } // namespace -std::shared_ptr BTreeIndexMeta::Deserialize(const std::shared_ptr& meta, - paimon::MemoryPool* pool) { +Result> BTreeIndexMeta::Deserialize( + const std::shared_ptr& meta, paimon::MemoryPool* pool) { + if (meta == nullptr) { + return Status::Invalid("Cannot deserialize BTree index metadata from a null buffer."); + } + if (pool == nullptr) { + return Status::Invalid("Cannot deserialize BTree index metadata with a null memory pool."); + } + // Legacy metadata contains two int32 lengths and one has-nulls byte. + constexpr size_t kMinimumMetadataSize = 2 * sizeof(int32_t) + sizeof(int8_t); + if (meta->size() < kMinimumMetadataSize) { + return Status::Invalid(fmt::format( + "BTree index metadata is truncated: expected at least {} bytes, but found {}.", + kMinimumMetadataSize, meta->size())); + } + if (meta->size() > static_cast(std::numeric_limits::max())) { + return Status::Invalid( + fmt::format("BTree index metadata size {} exceeds the supported maximum {}.", + meta->size(), std::numeric_limits::max())); + } + MemorySlice slice = MemorySlice::Wrap(meta); MemorySliceInput input = slice.ToInput(); int32_t first_key_len = input.ReadInt(); - std::shared_ptr first_key = ReadKey(&input, first_key_len, pool); + constexpr int32_t kRequiredAfterFirstKey = sizeof(int32_t) + sizeof(int8_t); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr first_key, + ReadKey(&input, first_key_len, kRequiredAfterFirstKey, "first key", pool)); int32_t last_key_len = input.ReadInt(); - std::shared_ptr last_key = ReadKey(&input, last_key_len, pool); - bool has_nulls = input.ReadByte() == static_cast(1); + constexpr int32_t kRequiredAfterLastKey = sizeof(int8_t); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr last_key, + ReadKey(&input, last_key_len, kRequiredAfterLastKey, "last key", pool)); + int8_t has_nulls_byte = input.ReadByte(); + if (has_nulls_byte != 0 && has_nulls_byte != 1) { + return Status::Invalid( + fmt::format("BTree index metadata has invalid has-nulls value {}.", has_nulls_byte)); + } + bool has_nulls = has_nulls_byte == 1; - if (input.Available() >= 2) { + if (input.Available() == 2) { int8_t format_version = input.ReadByte(); - if (format_version == kFormatVersionWithNullFlags) { - int8_t null_key_flags = input.ReadByte(); - if ((null_key_flags & kFirstKeyIsNull) != 0) { - first_key.reset(); + if (format_version != kFormatVersionWithNullFlags) { + return Status::Invalid( + fmt::format("Unsupported BTree index metadata version {}.", format_version)); + } + int8_t null_key_flags = input.ReadByte(); + constexpr int8_t kKnownNullFlags = kFirstKeyIsNull | kLastKeyIsNull; + if ((null_key_flags & ~kKnownNullFlags) != 0) { + return Status::Invalid( + fmt::format("BTree index metadata has invalid null-key flags {}.", null_key_flags)); + } + if ((null_key_flags & kFirstKeyIsNull) != 0) { + if (first_key_len != 0) { + return Status::Invalid("BTree index metadata marks a non-empty first key as null."); } - if ((null_key_flags & kLastKeyIsNull) != 0) { - last_key.reset(); + first_key.reset(); + } + if ((null_key_flags & kLastKeyIsNull) != 0) { + if (last_key_len != 0) { + return Status::Invalid("BTree index metadata marks a non-empty last key as null."); } + last_key.reset(); } - } else if (first_key_len == 0 && last_key_len == 0 && has_nulls) { + } else if (input.Available() == 0 && first_key_len == 0 && last_key_len == 0 && has_nulls) { // Legacy metadata used zero length for null keys. Both empty boundaries plus a null bitmap // identify an all-null file; a single empty boundary remains a valid serialized key. first_key.reset(); last_key.reset(); + } else if (input.Available() != 0) { + return Status::Invalid(fmt::format("BTree index metadata has {} unexpected trailing bytes.", + input.Available())); } return std::make_shared(first_key, last_key, has_nulls); } diff --git a/src/paimon/common/global_index/btree/btree_index_meta.h b/src/paimon/common/global_index/btree/btree_index_meta.h index 85df5ff30..eb8158f12 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.h +++ b/src/paimon/common/global_index/btree/btree_index_meta.h @@ -24,6 +24,7 @@ #include "paimon/common/memory/memory_slice_input.h" #include "paimon/memory/bytes.h" +#include "paimon/result.h" namespace paimon { /// Index metadata for each BTree index file. @@ -31,8 +32,8 @@ namespace paimon { /// Empty serialized keys are valid, so null boundary keys are encoded separately with flags. class BTreeIndexMeta { public: - static std::shared_ptr Deserialize(const std::shared_ptr& meta, - paimon::MemoryPool* pool); + static Result> Deserialize(const std::shared_ptr& meta, + paimon::MemoryPool* pool); std::shared_ptr Serialize(paimon::MemoryPool* pool) const; public: diff --git a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp b/src/paimon/common/global_index/btree/btree_index_meta_test.cpp index 70c4dfcb4..b0d78fa3b 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp +++ b/src/paimon/common/global_index/btree/btree_index_meta_test.cpp @@ -19,9 +19,14 @@ #include "paimon/common/global_index/btree/btree_index_meta.h" +#include +#include +#include + #include "gtest/gtest.h" #include "paimon/common/memory/memory_slice_output.h" #include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" namespace paimon::test { class BTreeIndexMetaTest : public ::testing::Test { @@ -48,6 +53,28 @@ class BTreeIndexMetaTest : public ::testing::Test { return output.ToSlice().CopyBytes(pool_.get()); } + std::shared_ptr MetadataBytes(int32_t first_key_length, const std::string& first_key, + int32_t last_key_length, const std::string& last_key, + int8_t has_nulls, + const std::vector& suffix = {}) const { + MemorySliceOutput output( + static_cast(first_key.size() + last_key.size() + suffix.size() + 9), + pool_.get()); + output.WriteValue(first_key_length); + if (!first_key.empty()) { + output.WriteBytes(std::make_shared(first_key, pool_.get())); + } + output.WriteValue(last_key_length); + if (!last_key.empty()) { + output.WriteBytes(std::make_shared(last_key, pool_.get())); + } + output.WriteValue(has_nulls); + for (int8_t byte : suffix) { + output.WriteValue(byte); + } + return output.ToSlice().CopyBytes(pool_.get()); + } + std::shared_ptr pool_; }; @@ -62,7 +89,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeNormalKeys) { ASSERT_GT(serialized->size(), 0u); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key @@ -88,7 +116,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstKey) { auto serialized = meta->Serialize(pool_.get()); ASSERT_EQ(serialized->size(), 11 + last_key->size()); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -105,7 +134,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstAndLastKeysWithNulls) { auto serialized = meta->Serialize(pool_.get()); ASSERT_EQ(serialized->size(), 11); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -120,7 +150,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeOnlyNulls) { auto serialized = meta->Serialize(pool_.get()); ASSERT_EQ(serialized->size(), 11); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); ASSERT_FALSE(deserialized->FirstKey()); ASSERT_FALSE(deserialized->LastKey()); @@ -132,7 +163,8 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyOnlyNulls) { auto empty_key = std::make_shared(0, pool_.get()); auto serialized = LegacyMetaBytes(empty_key, empty_key, true); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_FALSE(deserialized->FirstKey()); ASSERT_FALSE(deserialized->LastKey()); ASSERT_TRUE(deserialized->HasNulls()); @@ -144,7 +176,8 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstKey) { auto last_key = std::make_shared("last_key_data", pool_.get()); auto serialized = LegacyMetaBytes(empty_key, last_key, false); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -158,7 +191,8 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) { auto empty_key = std::make_shared(0, pool_.get()); auto serialized = LegacyMetaBytes(empty_key, empty_key, false); - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized->FirstKey()); ASSERT_EQ(deserialized->FirstKey()->size(), 0); ASSERT_TRUE(deserialized->LastKey()); @@ -167,6 +201,52 @@ TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) { ASSERT_FALSE(deserialized->OnlyNulls()); } +TEST_F(BTreeIndexMetaTest, DeserializeRejectsNullAndTruncatedMetadata) { + ASSERT_NOK(BTreeIndexMeta::Deserialize(nullptr, pool_.get())); + for (size_t size = 0; size < 9; size++) { + auto truncated = std::make_shared(std::string(size, '\0'), pool_.get()); + ASSERT_NOK(BTreeIndexMeta::Deserialize(truncated, pool_.get())); + } +} + +TEST_F(BTreeIndexMetaTest, DeserializeRejectsInvalidKeyLengths) { + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(/*first_key_length=*/-1, "", /*last_key_length=*/0, "", + /*has_nulls=*/0), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(/*first_key_length=*/100, "", /*last_key_length=*/0, "", + /*has_nulls=*/0), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(/*first_key_length=*/0, "", /*last_key_length=*/-1, "", + /*has_nulls=*/0), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(/*first_key_length=*/0, "", /*last_key_length=*/100, "", + /*has_nulls=*/0), + pool_.get())); +} + +TEST_F(BTreeIndexMetaTest, DeserializeRejectsMalformedFlagsAndTrailingBytes) { + ASSERT_NOK( + BTreeIndexMeta::Deserialize(MetadataBytes(0, "", 0, "", /*has_nulls=*/2), pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*truncated_version=*/1}), pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*unsupported_version=*/2, /*flags=*/0}), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*version=*/1, /*unknown_flags=*/4}), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(0, "", 0, "", /*has_nulls=*/0, {/*version=*/1, /*flags=*/0, /*trailing=*/0}), + pool_.get())); + ASSERT_NOK(BTreeIndexMeta::Deserialize( + MetadataBytes(1, "x", 0, "", /*has_nulls=*/0, {/*version=*/1, /*first_key_is_null=*/1}), + pool_.get())); +} + TEST_F(BTreeIndexMetaTest, HasNullsAndOnlyNulls) { // Case 1: Has nulls with keys auto meta1 = @@ -204,7 +284,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeNoNulls) { ASSERT_TRUE(serialized); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify has_nulls is false @@ -221,7 +302,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyFirstKey) { ASSERT_TRUE(serialized); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key @@ -243,7 +325,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeWithOnlyLastKey) { ASSERT_TRUE(serialized); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key is null @@ -268,7 +351,8 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeBinaryKeys) { ASSERT_TRUE(serialized); // Deserialize - auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr deserialized, + BTreeIndexMeta::Deserialize(serialized, pool_.get())); ASSERT_TRUE(deserialized); // Verify first_key diff --git a/src/paimon/common/global_index/btree/key_serializer.cpp b/src/paimon/common/global_index/btree/key_serializer.cpp index 1ce1f6b4c..464f37ea4 100644 --- a/src/paimon/common/global_index/btree/key_serializer.cpp +++ b/src/paimon/common/global_index/btree/key_serializer.cpp @@ -22,15 +22,107 @@ #include "fmt/format.h" #include "paimon/common/memory/memory_slice_input.h" #include "paimon/common/memory/memory_slice_output.h" +#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/field_type_utils.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/preconditions.h" +#include "paimon/common/utils/var_length_int_utils.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/status.h" namespace paimon { +namespace { + +Status ValidateExactLength(const MemorySlice& slice, const std::shared_ptr& type, + int32_t expected_length) { + if (slice.Length() != expected_length) { + return Status::Invalid( + fmt::format("Invalid serialized {} key length: expected {}, but found {}.", + type->ToString(), expected_length, slice.Length())); + } + return Status::OK(); +} + +Status ValidateNonCompactTimestamp(const MemorySlice& slice) { + constexpr int32_t kMillisLength = sizeof(int64_t); + constexpr int32_t kMinimumLength = kMillisLength + 1; + // nano-of-millisecond is at most 999,999, which uses no more than three varint bytes. + constexpr int32_t kMaximumNanosLength = 3; + constexpr int32_t kMaximumLength = kMillisLength + kMaximumNanosLength; + if (slice.Length() < kMinimumLength || slice.Length() > kMaximumLength) { + return Status::Invalid(fmt::format( + "Invalid serialized timestamp key length: expected between {} and {}, but found {}.", + kMinimumLength, kMaximumLength, slice.Length())); + } + + int32_t terminal_position = -1; + for (int32_t position = kMillisLength; position < slice.Length(); ++position) { + auto byte = static_cast(slice.Data()[position]); + if ((byte & 0x80) == 0) { + terminal_position = position; + break; + } + } + if (terminal_position == -1) { + return Status::Invalid("Serialized timestamp key contains an unterminated nanos varint."); + } + if (terminal_position != slice.Length() - 1) { + return Status::Invalid("Serialized timestamp key contains trailing bytes."); + } + + int32_t offset = kMillisLength; + PAIMON_ASSIGN_OR_RAISE(int32_t nanos, VarLengthIntUtils::DecodeInt(slice.Data(), &offset)); + if (nanos > 999999) { + return Status::Invalid( + fmt::format("Serialized timestamp key has invalid nanos value {}.", nanos)); + } + return Status::OK(); +} + +Status ValidateNonCompactDecimal(const MemorySlice& slice) { + constexpr int32_t kMaximumLength = sizeof(Decimal::int128_t); + if (slice.Length() < 1 || slice.Length() > kMaximumLength) { + return Status::Invalid(fmt::format( + "Invalid serialized decimal key length: expected between 1 and {}, but found {}.", + kMaximumLength, slice.Length())); + } + if (slice.Length() == 1) { + return Status::OK(); + } + + auto first = static_cast(slice.Data()[0]); + auto second = static_cast(slice.Data()[1]); + if ((first == 0 && (second & 0x80) == 0) || (first == 0xFF && (second & 0x80) != 0)) { + return Status::Invalid("Serialized decimal key contains redundant sign-extension bytes."); + } + return Status::OK(); +} + +Status ValidateDecimal(const MemorySlice& slice, + const std::shared_ptr& type) { + arrow::Decimal128 value; + if (Decimal::IsCompact(type->precision())) { + PAIMON_RETURN_NOT_OK(ValidateExactLength(slice, type, sizeof(int64_t))); + value = arrow::Decimal128(slice.ReadLong(0)); + } else { + PAIMON_RETURN_NOT_OK(ValidateNonCompactDecimal(slice)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + arrow::Decimal128 decoded, + arrow::Decimal128::FromBigEndian(reinterpret_cast(slice.Data()), + slice.Length())); + value = decoded; + } + if (!value.FitsInPrecision(type->precision())) { + return Status::Invalid( + fmt::format("Serialized decimal key does not fit precision {}.", type->precision())); + } + return Status::OK(); +} + +} // namespace + Result> KeySerializer::SerializeKey( const Literal& literal, const std::shared_ptr& type, MemoryPool* pool) { if (literal.IsNull()) { @@ -139,6 +231,7 @@ Result> KeySerializer::SerializeKey( Result KeySerializer::DeserializeKey(const MemorySlice& slice, const std::shared_ptr& type, MemoryPool* pool) { + PAIMON_RETURN_NOT_OK(ValidateSerializedKey(slice, type)); switch (type->id()) { case arrow::Type::type::BOOL: return Literal(slice.ReadByte(0) == 1 ? true : false); @@ -196,6 +289,51 @@ Result KeySerializer::DeserializeKey(const MemorySlice& slice, } } +Status KeySerializer::ValidateSerializedKey(const MemorySlice& slice, + const std::shared_ptr& type) { + if (type == nullptr) { + return Status::Invalid("Cannot validate a serialized BTree key without a key type."); + } + switch (type->id()) { + case arrow::Type::type::BOOL: { + PAIMON_RETURN_NOT_OK(ValidateExactLength(slice, type, sizeof(int8_t))); + auto value = static_cast(slice.Data()[0]); + if (value > 1) { + return Status::Invalid( + fmt::format("Invalid serialized boolean key value {}.", value)); + } + return Status::OK(); + } + case arrow::Type::type::INT8: + return ValidateExactLength(slice, type, sizeof(int8_t)); + case arrow::Type::type::INT16: + return ValidateExactLength(slice, type, sizeof(int16_t)); + case arrow::Type::type::INT32: + case arrow::Type::type::DATE32: + case arrow::Type::type::FLOAT: + return ValidateExactLength(slice, type, sizeof(int32_t)); + case arrow::Type::type::INT64: + case arrow::Type::type::DOUBLE: + return ValidateExactLength(slice, type, sizeof(int64_t)); + case arrow::Type::type::STRING: + return Status::OK(); + case arrow::Type::type::TIMESTAMP: { + auto timestamp_type = checked_pointer_cast(type); + if (Timestamp::IsCompact(DateTimeUtils::GetPrecisionFromType(timestamp_type))) { + return ValidateExactLength(slice, type, sizeof(int64_t)); + } + return ValidateNonCompactTimestamp(slice); + } + case arrow::Type::type::DECIMAL128: { + auto decimal_type = checked_pointer_cast(type); + return ValidateDecimal(slice, decimal_type); + } + default: + return Status::Invalid(fmt::format( + "Not support validate serialized {} type in BTreeGlobalIndex", type->ToString())); + } +} + MemorySlice::SliceComparator KeySerializer::CreateComparator( const std::shared_ptr& type, const std::shared_ptr& pool) { // Fast paths for integer and string types: direct value comparison without Literal diff --git a/src/paimon/common/global_index/btree/key_serializer.h b/src/paimon/common/global_index/btree/key_serializer.h index 82f851030..d44b5bdc1 100644 --- a/src/paimon/common/global_index/btree/key_serializer.h +++ b/src/paimon/common/global_index/btree/key_serializer.h @@ -38,6 +38,9 @@ class KeySerializer { const std::shared_ptr& type, MemoryPool* pool); + static Status ValidateSerializedKey(const MemorySlice& slice, + const std::shared_ptr& type); + static MemorySlice::SliceComparator CreateComparator( const std::shared_ptr& type, const std::shared_ptr& pool); }; diff --git a/src/paimon/common/global_index/btree/key_serializer_test.cpp b/src/paimon/common/global_index/btree/key_serializer_test.cpp index 3dc8a0f68..e36e72ed6 100644 --- a/src/paimon/common/global_index/btree/key_serializer_test.cpp +++ b/src/paimon/common/global_index/btree/key_serializer_test.cpp @@ -208,6 +208,42 @@ TEST_F(KeySerializerTest, SerializeAndDeserializeAllTypes) { } } +TEST_F(KeySerializerTest, RejectsMalformedSerializedKeys) { + auto wrap = [this](const std::string& value) { + return MemorySlice::Wrap(std::make_shared(value, pool_.get())); + }; + + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(std::string(3, '\0')), arrow::int32())); + ASSERT_NOK( + KeySerializer::DeserializeKey(wrap(std::string(3, '\0')), arrow::int32(), pool_.get())); + + std::string invalid_boolean(1, static_cast(2)); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(invalid_boolean), arrow::boolean())); + + auto nanos_timestamp = arrow::timestamp(arrow::TimeUnit::NANO); + std::string unterminated_timestamp(9, '\0'); + unterminated_timestamp[8] = static_cast(0x80); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(unterminated_timestamp), nanos_timestamp)); + + std::string out_of_range_timestamp(11, '\0'); + out_of_range_timestamp[8] = static_cast(0xC0); + out_of_range_timestamp[9] = static_cast(0x84); + out_of_range_timestamp[10] = static_cast(0x3D); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(out_of_range_timestamp), nanos_timestamp)); + + auto non_compact_decimal = arrow::decimal128(25, 3); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(wrap(""), non_compact_decimal)); + ASSERT_NOK( + KeySerializer::ValidateSerializedKey(wrap(std::string(17, '\0')), non_compact_decimal)); + + auto narrow_decimal = arrow::decimal128(1, 0); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out_of_range_decimal, + KeySerializer::SerializeKey(Literal(Decimal::FromUnscaledLong(10, 1, 0)), + narrow_decimal, pool_.get())); + ASSERT_NOK(KeySerializer::ValidateSerializedKey(MemorySlice::Wrap(out_of_range_decimal), + narrow_decimal)); +} + TEST_F(KeySerializerTest, CreateComparator) { // INT comparator { diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp index bc66e17a1..4ea7576b6 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp @@ -38,35 +38,47 @@ #include "paimon/utils/roaring_bitmap64.h" namespace paimon { -LazyFilteredBTreeReader::LazyFilteredBTreeReader( +Result> LazyFilteredBTreeReader::Create( std::optional read_buffer_size, const std::vector& files, const std::shared_ptr& key_type, const std::shared_ptr& file_reader, const std::shared_ptr& cache_manager, const std::shared_ptr& pool, - const std::shared_ptr& executor) + const std::shared_ptr& executor) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_selector, + BTreeFileMetaSelector::Create(files, key_type, pool)); + return std::shared_ptr( + new LazyFilteredBTreeReader(read_buffer_size, std::move(file_selector), key_type, + file_reader, cache_manager, pool, executor)); +} + +LazyFilteredBTreeReader::LazyFilteredBTreeReader( + std::optional read_buffer_size, std::unique_ptr file_selector, + std::shared_ptr key_type, std::shared_ptr file_reader, + std::shared_ptr cache_manager, std::shared_ptr pool, + std::shared_ptr executor) : read_buffer_size_(read_buffer_size), - pool_(pool), - file_selector_(files, key_type, pool), - key_type_(key_type), - file_reader_(file_reader), - cache_manager_(cache_manager), - executor_(executor) {} + pool_(std::move(pool)), + file_selector_(std::move(file_selector)), + key_type_(std::move(key_type)), + file_reader_(std::move(file_reader)), + cache_manager_(std::move(cache_manager)), + executor_(std::move(executor)) {} Result> LazyFilteredBTreeReader::VisitIsNotNull() { return DispatchVisit( - [this]() { return file_selector_.VisitIsNotNull(); }, + [this]() { return file_selector_->VisitIsNotNull(); }, [](const std::shared_ptr& reader) { return reader->VisitIsNotNull(); }); } Result> LazyFilteredBTreeReader::VisitIsNull() { return DispatchVisit( - [this]() { return file_selector_.VisitIsNull(); }, + [this]() { return file_selector_->VisitIsNull(); }, [](const std::shared_ptr& reader) { return reader->VisitIsNull(); }); } Result> LazyFilteredBTreeReader::VisitEqual( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitEqual(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitEqual(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitEqual(literal); }); @@ -74,7 +86,7 @@ Result> LazyFilteredBTreeReader::VisitEqual( Result> LazyFilteredBTreeReader::VisitNotEqual( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitNotEqual(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitNotEqual(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitNotEqual(literal); }); @@ -82,7 +94,7 @@ Result> LazyFilteredBTreeReader::VisitNotEqua Result> LazyFilteredBTreeReader::VisitLessThan( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitLessThan(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitLessThan(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitLessThan(literal); }); @@ -90,7 +102,7 @@ Result> LazyFilteredBTreeReader::VisitLessTha Result> LazyFilteredBTreeReader::VisitLessOrEqual( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitLessOrEqual(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitLessOrEqual(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitLessOrEqual(literal); }); @@ -98,7 +110,7 @@ Result> LazyFilteredBTreeReader::VisitLessOrE Result> LazyFilteredBTreeReader::VisitGreaterThan( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitGreaterThan(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitGreaterThan(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitGreaterThan(literal); }); @@ -106,15 +118,16 @@ Result> LazyFilteredBTreeReader::VisitGreater Result> LazyFilteredBTreeReader::VisitGreaterOrEqual( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitGreaterOrEqual(literal); }, - [&literal](const std::shared_ptr& reader) { - return reader->VisitGreaterOrEqual(literal); - }); + return DispatchVisit( + [this, &literal]() { return file_selector_->VisitGreaterOrEqual(literal); }, + [&literal](const std::shared_ptr& reader) { + return reader->VisitGreaterOrEqual(literal); + }); } Result> LazyFilteredBTreeReader::VisitIn( const std::vector& literals) { - return DispatchVisit([this, &literals]() { return file_selector_.VisitIn(literals); }, + return DispatchVisit([this, &literals]() { return file_selector_->VisitIn(literals); }, [&literals](const std::shared_ptr& reader) { return reader->VisitIn(literals); }); @@ -122,7 +135,7 @@ Result> LazyFilteredBTreeReader::VisitIn( Result> LazyFilteredBTreeReader::VisitNotIn( const std::vector& literals) { - return DispatchVisit([this, &literals]() { return file_selector_.VisitNotIn(literals); }, + return DispatchVisit([this, &literals]() { return file_selector_->VisitNotIn(literals); }, [&literals](const std::shared_ptr& reader) { return reader->VisitNotIn(literals); }); @@ -130,7 +143,7 @@ Result> LazyFilteredBTreeReader::VisitNotIn( Result> LazyFilteredBTreeReader::VisitStartsWith( const Literal& prefix) { - return DispatchVisit([this, &prefix]() { return file_selector_.VisitStartsWith(prefix); }, + return DispatchVisit([this, &prefix]() { return file_selector_->VisitStartsWith(prefix); }, [&prefix](const std::shared_ptr& reader) { return reader->VisitStartsWith(prefix); }); @@ -138,7 +151,7 @@ Result> LazyFilteredBTreeReader::VisitStartsW Result> LazyFilteredBTreeReader::VisitEndsWith( const Literal& suffix) { - return DispatchVisit([this, &suffix]() { return file_selector_.VisitEndsWith(suffix); }, + return DispatchVisit([this, &suffix]() { return file_selector_->VisitEndsWith(suffix); }, [&suffix](const std::shared_ptr& reader) { return reader->VisitEndsWith(suffix); }); @@ -146,7 +159,7 @@ Result> LazyFilteredBTreeReader::VisitEndsWit Result> LazyFilteredBTreeReader::VisitContains( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitContains(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitContains(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitContains(literal); }); @@ -154,7 +167,7 @@ Result> LazyFilteredBTreeReader::VisitContain Result> LazyFilteredBTreeReader::VisitLike( const Literal& literal) { - return DispatchVisit([this, &literal]() { return file_selector_.VisitLike(literal); }, + return DispatchVisit([this, &literal]() { return file_selector_->VisitLike(literal); }, [&literal](const std::shared_ptr& reader) { return reader->VisitLike(literal); }); @@ -214,7 +227,8 @@ Result> LazyFilteredBTreeReader::CreateSingle auto comparator = KeySerializer::CreateComparator(key_type_, pool_); // Get min/max key slices from meta data (keep as slices; Create() will deserialize) - auto index_meta = BTreeIndexMeta::Deserialize(meta.metadata, pool_.get()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr index_meta, + BTreeIndexMeta::Deserialize(meta.metadata, pool_.get())); std::optional min_key_slice; std::optional max_key_slice; if (index_meta->FirstKey()) { diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h index 0603e78cf..74ed978b8 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h @@ -40,13 +40,12 @@ namespace paimon { class LazyFilteredBTreeReader : public GlobalIndexReader { public: - LazyFilteredBTreeReader(std::optional read_buffer_size, - const std::vector& files, - const std::shared_ptr& key_type, - const std::shared_ptr& file_reader, - const std::shared_ptr& cache_manager, - const std::shared_ptr& pool, - const std::shared_ptr& executor); + static Result> Create( + std::optional read_buffer_size, const std::vector& files, + const std::shared_ptr& key_type, + const std::shared_ptr& file_reader, + const std::shared_ptr& cache_manager, const std::shared_ptr& pool, + const std::shared_ptr& executor); Result> VisitIsNotNull() override; Result> VisitIsNull() override; @@ -80,6 +79,13 @@ class LazyFilteredBTreeReader : public GlobalIndexReader { } private: + LazyFilteredBTreeReader(std::optional read_buffer_size, + std::unique_ptr file_selector, + std::shared_ptr key_type, + std::shared_ptr file_reader, + std::shared_ptr cache_manager, + std::shared_ptr pool, std::shared_ptr executor); + using SelectAction = std::function>()>; using ReaderAction = std::function>( const std::shared_ptr&)>; @@ -96,7 +102,7 @@ class LazyFilteredBTreeReader : public GlobalIndexReader { private: std::optional read_buffer_size_; std::shared_ptr pool_; - BTreeFileMetaSelector file_selector_; + std::unique_ptr file_selector_; std::shared_ptr key_type_; std::shared_ptr file_reader_; std::shared_ptr cache_manager_; diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp index bf5503fa7..a1be0108b 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp @@ -139,9 +139,11 @@ class LazyFilteredBTreeReaderTest : public ::testing::Test { const std::shared_ptr& executor = nullptr) const { auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); - return std::make_shared(/*read_buffer_size=*/std::nullopt, - all_metas_, arrow::int32(), file_reader, - cache_manager, pool_, executor); + EXPECT_OK_AND_ASSIGN(std::shared_ptr reader, + LazyFilteredBTreeReader::Create( + /*read_buffer_size=*/std::nullopt, all_metas_, arrow::int32(), + file_reader, cache_manager, pool_, executor)); + return reader; } void CheckResult(const std::shared_ptr& result, @@ -299,6 +301,23 @@ TEST_F(LazyFilteredBTreeReaderTest, TestVisitNotIn) { CheckResult(result, {3, 4, 6, 7, 10}); } +TEST_F(LazyFilteredBTreeReaderTest, RejectsMismatchedLiteralEncoding) { + auto reader = CreateReader(); + Literal int8_literal(static_cast(1)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr not_equal, + reader->VisitNotEqual(int8_literal)); + auto not_equal_bitmap = std::dynamic_pointer_cast(not_equal); + ASSERT_TRUE(not_equal_bitmap != nullptr); + ASSERT_NOK(not_equal_bitmap->GetBitmap()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr not_in, + reader->VisitNotIn({int8_literal})); + auto not_in_bitmap = std::dynamic_pointer_cast(not_in); + ASSERT_TRUE(not_in_bitmap != nullptr); + ASSERT_NOK(not_in_bitmap->GetBitmap()); +} + // --- VisitIsNull --- TEST_F(LazyFilteredBTreeReaderTest, TestVisitIsNull) { @@ -364,9 +383,11 @@ TEST_F(LazyFilteredBTreeReaderTest, TestEmptyFilesList) { std::vector empty_metas; auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); - auto reader = std::make_shared( - /*read_buffer_size=*/std::nullopt, empty_metas, arrow::int32(), file_reader, cache_manager, - pool_, /*executor=*/nullptr); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr reader, + LazyFilteredBTreeReader::Create(/*read_buffer_size=*/std::nullopt, empty_metas, + arrow::int32(), file_reader, cache_manager, pool_, + /*executor=*/nullptr)); // Any query on empty files should return empty bitmap Literal literal_1(1); @@ -456,12 +477,23 @@ TEST_F(LazyFilteredBTreeReaderTest, TestParallelEmptyFilesList) { std::vector empty_metas; auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); - auto reader = std::make_shared( - /*read_buffer_size=*/std::nullopt, empty_metas, arrow::int32(), file_reader, cache_manager, - pool_, executor); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + LazyFilteredBTreeReader::Create(/*read_buffer_size=*/std::nullopt, + empty_metas, arrow::int32(), file_reader, + cache_manager, pool_, executor)); Literal literal_1(1); ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(literal_1)); CheckEmpty(result); } +TEST_F(LazyFilteredBTreeReaderTest, RejectsMalformedFileMetadata) { + auto file_reader = std::make_shared(fs_, base_path_); + auto cache_manager = std::make_shared(1024 * 1024, 0.5); + std::vector invalid_metas = { + GlobalIndexIOMeta("invalid", 1, /*metadata=*/nullptr)}; + ASSERT_NOK(LazyFilteredBTreeReader::Create( + /*read_buffer_size=*/std::nullopt, invalid_metas, arrow::int32(), file_reader, + cache_manager, pool_, /*executor=*/nullptr)); +} + } // namespace paimon::test diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index a1effa8c4..11fdec251 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -156,7 +156,14 @@ Result> RawFileSplitRead::CreateReader( Result RawFileSplitRead::Match(const std::shared_ptr& split, bool force_keep_delete) const { - auto split_impl = dynamic_cast(split.get()); + bool is_indexed = false; + std::shared_ptr data_split = split; + if (auto indexed_split = std::dynamic_pointer_cast(split)) { + PAIMON_RETURN_NOT_OK(indexed_split->Validate()); + data_split = indexed_split->GetDataSplit(); + is_indexed = true; + } + auto split_impl = dynamic_cast(data_split.get()); if (split_impl == nullptr) { return Status::Invalid("unexpected error, split cast to impl failed"); } @@ -164,14 +171,16 @@ Result RawFileSplitRead::Match(const std::shared_ptr& split, // for append table, always return true return true; } - bool matched = !force_keep_delete && !split_impl->IsStreaming() && split_impl->RawConvertible(); + bool matched = !force_keep_delete && !split_impl->IsStreaming() && + (is_indexed || split_impl->RawConvertible()); if (matched) { // for legacy version, we are not sure if there are delete rows, but in order to be // compatible with the query acceleration of the OLAP engine, we have generated raw // files. // Here, for the sake of correctness, we still need to perform drop delete filtering. for (const auto& file : split_impl->DataFiles()) { - if (file->delete_row_count == std::nullopt) { + if (file == nullptr || file->delete_row_count == std::nullopt || + file->delete_row_count.value() != 0) { return false; } } diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 29dc0f561..6f7ae9781 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -442,8 +442,9 @@ TEST_F(RawFileSplitReadTest, TestMatch) { CreateDefaultExecutor(/*thread_count=*/2)); auto split_read = std::make_unique( /*path_factory=*/nullptr, std::move(internal_context), pool_, executor); - auto create_data_split = [this](bool is_streaming, - bool raw_convertible) -> std::shared_ptr { + auto create_data_split = [this](bool is_streaming, bool raw_convertible, + std::optional delete_row_count = + 0) -> std::shared_ptr { auto meta = std::make_shared( "data-d7725088-6bd4-4e70-9ce6-714ae93b47cc-0.orc", /*file_size=*/863, /*row_count=*/1, /*min_key=*/BinaryRowGenerator::GenerateRow({std::string("Alice"), 1}, pool_.get()), @@ -457,8 +458,8 @@ TEST_F(RawFileSplitReadTest, TestMatch) { pool_.get()), /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), - /*creation_time=*/Timestamp(1743525392885ll, 0), - /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*creation_time=*/Timestamp(1743525392885ll, 0), delete_row_count, + /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); @@ -523,8 +524,33 @@ TEST_F(RawFileSplitReadTest, TestMatch) { "Invalid file-local row range [0, 1]"); } { - ASSERT_NOK(split_read->Match(nullptr, /*force_keep_delete=*/false)); + auto data_split = std::dynamic_pointer_cast(create_data_split( + /*is_streaming=*/false, /*raw_convertible=*/false, /*delete_row_count=*/std::nullopt)); + auto indexed_split = + std::make_shared(data_split, std::vector{Range(0, 0)}); + ASSERT_OK_AND_ASSIGN(bool match_result, + split_read->Match(indexed_split, /*force_keep_delete=*/false)); + ASSERT_FALSE(match_result); + } + { + auto data_split = std::dynamic_pointer_cast(create_data_split( + /*is_streaming=*/false, /*raw_convertible=*/false, /*delete_row_count=*/1)); + auto indexed_split = + std::make_shared(data_split, std::vector{Range(0, 0)}); + ASSERT_OK_AND_ASSIGN(bool match_result, + split_read->Match(indexed_split, /*force_keep_delete=*/false)); + ASSERT_FALSE(match_result); + } + { + auto data_split = std::dynamic_pointer_cast(create_data_split( + /*is_streaming=*/false, /*raw_convertible=*/false, /*delete_row_count=*/0)); + auto indexed_split = + std::make_shared(data_split, std::vector{Range(0, 0)}); + ASSERT_OK_AND_ASSIGN(bool match_result, + split_read->Match(indexed_split, /*force_keep_delete=*/false)); + ASSERT_TRUE(match_result); } + ASSERT_NOK(split_read->Match(nullptr, /*force_keep_delete=*/false)); } } // namespace paimon::test diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp index f3a1b7569..208807493 100644 --- a/src/paimon/core/table/source/key_value_table_read.cpp +++ b/src/paimon/core/table/source/key_value_table_read.cpp @@ -88,13 +88,25 @@ Result> KeyValueTableRead::CreateReader( // of the inner split's raw-convertible marker, matching Java's dedicated provider. const std::shared_ptr& inner_split = indexed_split->GetDataSplit(); if (!force_keep_delete_) { + bool has_raw_reader = false; for (const auto& read : split_reads_) { if (dynamic_cast(read.get()) != nullptr) { - return read->CreateReader(indexed_split); + has_raw_reader = true; + PAIMON_ASSIGN_OR_RAISE(bool matched, + read->Match(indexed_split, /*force_keep_delete=*/false)); + if (matched) { + return read->CreateReader(indexed_split); + } + // A manually supplied or deserialized indexed split can still reference + // legacy files. Preserve merge semantics when raw-read safety is uncertain. + dispatch_split = inner_split; + break; } } - return Status::Invalid( - "create reader failed, primary-key indexed split has no raw reader."); + if (!has_raw_reader) { + return Status::Invalid( + "create reader failed, primary-key indexed split has no raw reader."); + } } else { // Keeping delete rows is incompatible with physical-position pruning. Reading the // inner split through the normal merge path preserves correctness. diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp index 325288e2b..07796b3f0 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -19,6 +19,7 @@ #include "paimon/core/table/source/primary_key_sorted_index_scan.h" +#include #include #include #include @@ -320,6 +321,14 @@ Result FindSourceIndex(const PkSortedIndexGroup& group, const DataFileMe return Status::Invalid(fmt::format( "Data file {} is not covered by its sorted-index source group.", data_file.file_name)); } + +bool SupportsIndexedRawRead(const DataSplitImpl& split) { + return std::all_of(split.DataFiles().begin(), split.DataFiles().end(), + [](const std::shared_ptr& file) { + return file != nullptr && file->delete_row_count.has_value() && + file->delete_row_count.value() == 0; + }); +} } // namespace Result PrimaryKeySortedIndexScan::CreatePlan( @@ -410,10 +419,15 @@ Result PrimaryKeySortedIndexScan::CreatePlan( std::vector files; for (const std::shared_ptr& split : data_splits) { auto bucket_groups = groups_by_bucket.find(BucketKey(split->Partition(), split->Bucket())); + // Legacy metadata may omit delete_row_count even when the split generator marks a split + // raw-convertible. Keep the entire original split so that the normal read path can merge + // DELETE rows with records from the other files in the split. + const bool supports_indexed_raw_read = SupportsIndexedRawRead(*split); for (size_t file_index = 0; file_index < split->DataFiles().size(); file_index++) { const std::shared_ptr& data_file = split->DataFiles()[file_index]; std::map> groups; - if (bucket_groups != groups_by_bucket.end() && data_file != nullptr) { + if (supports_indexed_raw_read && bucket_groups != groups_by_bucket.end() && + data_file != nullptr) { auto source_groups = bucket_groups->second.find(data_file->file_name); if (source_groups != bucket_groups->second.end()) { groups = source_groups->second; diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index e75591cd7..a56469656 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -28,6 +29,8 @@ #include "arrow/api.h" #include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/common/global_index/btree/btree_index_meta.h" +#include "paimon/common/global_index/btree/key_serializer.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" @@ -206,15 +209,16 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { } std::shared_ptr MakeDataFile(const std::string& name, int64_t row_count, - int32_t level, const FileSource& file_source) { + int32_t level, const FileSource& file_source, + std::optional delete_row_count = 0) { return std::make_shared( name, /*file_size=*/1024, row_count, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0, /*max_sequence_number=*/row_count, /*schema_id=*/0, level, /*extra_files=*/std::vector>(), - /*creation_time=*/Timestamp(1721643142456LL, 0), - /*delete_row_count=*/0, /*embedded_index=*/nullptr, file_source, + /*creation_time=*/Timestamp(1721643142456LL, 0), delete_row_count, + /*embedded_index=*/nullptr, file_source, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); } @@ -494,6 +498,41 @@ TEST_F(PrimaryKeySortedIndexScanTest, NonRawConvertibleSplitPreserved) { ASSERT_EQ(splits[0].get(), split.get()); } +TEST_F(PrimaryKeySortedIndexScanTest, UnknownDeleteCountPreservesOriginalSplit) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact(), std::nullopt), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN(PrimaryKeySortedIndexScan::Plan plan, + PrimaryKeySortedIndexScan::CreatePlan(kSnapshotId, {split}, definitions_, + MakeEntries(payload))); + ASSERT_TRUE(plan.Files()[0].Groups().empty()); + ASSERT_TRUE(plan.Files()[1].Groups().empty()); + + for (int64_t value : {10, 11}) { + SCOPED_TRACE(value == 10 ? "non-empty index result" : "empty index result"); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(value), + PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + ASSERT_EQ(splits[0], split); + } +} + +TEST_F(PrimaryKeySortedIndexScanTest, NonzeroDeleteCountPreservesOriginalSplit) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact(), 1), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN( + std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(payload), PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + ASSERT_EQ(splits[0], split); +} + TEST_F(PrimaryKeySortedIndexScanTest, InvalidRowRangePayloadFallsBack) { ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); // Rebuild the payload metadata with a row range end beyond the source rows: the group @@ -515,6 +554,50 @@ TEST_F(PrimaryKeySortedIndexScanTest, InvalidRowRangePayloadFallsBack) { ASSERT_EQ(split, splits[0]); } +TEST_F(PrimaryKeySortedIndexScanTest, MalformedBTreeMetadataFallsBack) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); + const GlobalIndexMeta& meta = payload->GetGlobalIndexMeta().value(); + auto short_key = std::make_shared(std::string(1, '\0'), pool_.get()); + auto invalid_key_meta = std::make_shared(short_key, short_key, false); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_key, + KeySerializer::SerializeKey(Literal(static_cast(10)), + arrow::int64(), pool_.get())); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr last_key, + KeySerializer::SerializeKey(Literal(static_cast(1)), arrow::int64(), pool_.get())); + auto reversed_meta = std::make_shared(first_key, last_key, false); + auto only_first_meta = std::make_shared(first_key, /*last_key=*/nullptr, false); + auto only_last_meta = std::make_shared(/*first_key=*/nullptr, last_key, false); + auto empty_nonnull_meta = + std::make_shared(/*first_key=*/nullptr, /*last_key=*/nullptr, false); + std::vector> malformed_metadata = { + nullptr, + std::make_shared(std::string(4, '\0'), pool_.get()), + invalid_key_meta->Serialize(pool_.get()), + reversed_meta->Serialize(pool_.get()), + only_first_meta->Serialize(pool_.get()), + only_last_meta->Serialize(pool_.get()), + empty_nonnull_meta->Serialize(pool_.get())}; + for (const std::shared_ptr& index_meta : malformed_metadata) { + SCOPED_TRACE(index_meta == nullptr ? "missing metadata" + : fmt::format("metadata size {}", index_meta->size())); + auto broken_payload = std::make_shared( + payload->IndexType(), payload->FileName(), payload->FileSize(), payload->RowCount(), + std::nullopt, std::nullopt, + GlobalIndexMeta(meta.row_range_start, meta.row_range_end, meta.index_field_id, + meta.extra_field_ids, index_meta, meta.source_meta)); + std::shared_ptr split = + MakeSplit({MakeDataFile("a.parquet", kFileARows, 5, FileSource::Compact()), + MakeDataFile("b.parquet", kFileBRows, 5, FileSource::Compact())}, + /*raw_convertible=*/true); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + PlanEvaluateConvert({split}, MakeEntries(broken_payload), + PriceEqual(10), PayloadReaderFactory())); + ASSERT_EQ(splits.size(), 1); + ASSERT_EQ(splits[0], split); + } +} + TEST_F(PrimaryKeySortedIndexScanTest, OutOfRangePositionsFailAllCoveredFiles) { ASSERT_OK_AND_ASSIGN(std::shared_ptr payload, BuildPayload()); std::shared_ptr split =